{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 多层全连接神经网络"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 线性模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 一维线性回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from torch import nn, optim\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "use_cuda = torch.cuda.is_available()                  # gpu可用\n",
    "device = torch.device('cuda' if use_cuda else 'cpu')  # 优先使用gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFF9JREFUeJzt3X+U5XV93/Hna3d2AwiWdXetyI9dN5IfSiOwExxjjRT9g1qOxNS0pATFkz2cWq2amqStzfEHbWqS9mhM4eghoGK6MXKQU4nVnGJExLRDMoOA4nrqds0EKgnLMohYdBnm3T/u3e+ZDDM7u7Df+d479/k4Z8587/d+7r3v+z0z85rP5/v5fm6qCkmSANZ1XYAkaXAYCpKkhqEgSWoYCpKkhqEgSWoYCpKkhqEgDZAkf5nk1V3XodFlKEhDKkkleWHXdWhtMRSkFSQZ67oGabUYChpKSX40ycNJzu3ffn6Sh5Kcv0TbH+m3/XsL9j03yeNJti7R/vIkf5bkg0keBt7bf70vJjnQf53dSU7ut39Tkj9e8Pi9SW5YcPu+JGcv8z4uSzLTf95/t+i+85L8rySPJHkgyVVJNvbv+3K/2d1JHkvyT5NsSvLZJPuTzPa3TzuKwyoZChpOVfV/gH8N7E5yAvAx4ONV9aUl2v4Q+CPglxbs/kXgC1W1f5mXeCmwD3gu8JtAgPcDzwd+EjgdeG+/7W3AK5KsS3IKsAF4OUCSHcCJwD2LXyDJi4APA5f1n3czsPCP+JPArwBbgJcBrwL+Rf89/Wy/zUuq6sSq+hS93+ePAduAM4DHgauWeX/SkuLaRxpmSW4GXgAU8NP9AFiq3UuBG4FtVTWfZAr4naq6YYm2lwNXVtUZh3ndnwPeU1Xn9G/fB1wM/BhwAXA28AZ6f8xfV1WvXeI53g28qKou6d9+FjALvKaqvrBE+3cAr6yq1/VvF3BmVe1dpsazgVuratNy70NazLFSDbvfB24GrjgUCEleAXy+f/9MVb24qu5I8n3glUkeAF7Yf9xy7lt4I8lzgd8DXgGcRO+/8tkFTW4Dzu8/723AI8Ar6YXCbcu8xvMXvk5VfT/JgQWv+WPAB4Bx4AR6v6/TyxXc7zF9ELgQOBQEJyVZX1VPHua9Sg2HjzS0kpwI/C5wHb1x/+cAVNXt/SGVE6vqxQsecj29IaTLgBur6geHefrFXej39/f9VFU9u/88WXD/oVB4RX/7Nnqh8EqWD4UH6A1DHXo/J9AbQjrkw8A36fUGng28a9FrLvZO4MeBl/bbHxpiOtxjpL/FUNAw+xAwXVW7gP8OfGSF9n8AvI7eH/RPHOVrnQQ8BjyS5FTg1xbdfxvwD4Djq+p+4HZ6/7FvBr66zHPeCFyU5O/3TyBfyd/+nTwJeBR4LMlPAG9e9Pi/AXYsav94v8bnAO85urcoGQoaUkkupvdH95/3d/0r4Nwkly73mP4f6zvp/cd/+1G+5PuAc4Hv0gugmxY99/+mFxq3928/Su9E9Z8tN3RTVfcCbwH+kF6vYRa4f0GTXwX+GfA9esNkn1r0FO8Fru/PTvon9HpNxwMPAZPAnxzle5Q80azRkuSjwHeq6je6rkUaRJ5o1shIsh34eeCcbiuRBpfDRxoJSf498HXgP1XVt7uuRxpUDh9Jkhr2FCRJjaE7p7Bly5bavn1712VI0lCZnp5+qKqestbXYkMXCtu3b2dqaqrrMiRpqCSZOZJ2Dh9JkhqGgiSp0VooJDkuyZ8nuTvJvUnet0Sby/trv9/V/9rVVj2SpJW1eU7hh8AFVfVYkg3AV5J8vqomF7X7VFW9tcU6JElHqLVQqN4FEI/1b27of3lRhCQNsFbPKSRZn+Qu4EHglqq6Y4lm/zjJPUluTHL6EveT5IokU0mm9u9f7oOyJEnPVKuhUFVPVtXZ9D5i8LwkZy1q8sfA9qr6KeAL9Na7X+p5rqmq8aoa37p1xWm2krTmTM/McvWte5memV258TOwKtcpVNUjSb5Eb6njry/Yf2BBs98Hfns16pGkYTI9M8ul105ycG6ejWPr2L1rgp3b2vmU1TZnH21NcnJ/+3jg1fQ+RWphm1MW3HwtsKeteiRpWE3uO8DBuXnmC56Ym2dy34GVH/Q0tdlTOIXeB4Cspxc+N1TVZ5NcCUxV1c3A25K8FpgDHgYub7EeSRpKEzs2s3FsHU/MzbNhbB0TOzav/KCnaehWSR0fHy+XuZA0aqZnZpncd4CJHZuf1tBRkumqGl+p3dCtfSRJo2jntk2tnUdYyGUuJEkNQ0GS1DAUJEkNQ0GSlrBaF4sNGk80S9Iiq3mx2KCxpyBJi6zmxWKDxlCQpEUOXSy2PrR+sdigcfhIkhbZuW0Tu3dNPKOLxYaVoSBJS1iti8UGjcNHkqSGoSBJahgKkqSGoSBJahgKkqSGoSBJahgKkqSGoSBJahgKkqSGoSCtklFdilnDxWUupFUwyksxa7jYU5BWwSgvxazhYihIq2CUl2LWcHH4SFoFo7wUs4aLoSCtklFdilnDxeEjSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNVoLhSTHJfnzJHcnuTfJ+5Zo8yNJPpVkb5I7kmxvqx5J0sra7Cn8ELigql4CnA1cmGRiUZtfBmar6oXAB4HfbrEeSdIKWguF6nmsf3ND/6sWNbsYuL6/fSPwqiRpqyZJ0uG1ek4hyfokdwEPArdU1R2LmpwK3AdQVXPAd4GnLDSf5IokU0mm9u/f32bJkjTSWg2Fqnqyqs4GTgPOS3LWoiZL9QoW9yaoqmuqaryqxrdu3dpGqZIkVmn2UVU9AnwJuHDRXfcDpwMkGQP+DvDwatQkSXqqNmcfbU1ycn/7eODVwDcXNbsZeGN/+/XAF6vqKT0FSdLqaPOT104Brk+ynl743FBVn01yJTBVVTcD1wF/kGQvvR7CJS3WI0laQWuhUFX3AOcssf/dC7Z/APxCWzVIko6OVzRLkhqGgqRlTc/McvWte5meme26FK2SNs8pSBpi0zOzXHrtJAfn5tk4to7duybYuW1T12WpZfYUJC1pct8BDs7NM1/wxNw8k/sOdF2SVoGhIGlJEzs2s3FsHesDG8bWMbHjKYsNaA1y+EjSknZu28TuXRNM7jvAxI7NDh2NCENB0rJ2bttkGIwYh4+kAeSsH3XFnoI0YJz1oy7ZU5AGjLN+1CVDQRowzvpRlxw+kgaMs37UJUNBGkDO+lFXHD7SipwJI40Oewo6LGfCSKPFnoIOy5kw0mgxFHRYzoSRRovDRzosZ8JIo8VQ0IqcCSONDoePJEkNQ0GS1DAUJEkNQ0GS1DAUJEkNQ0GS1DAUJEkNQ0GS1DAUJEkNQ0E6DJcN16hxmQtpGS4brlFkT0FahsuGaxQZCtIyXDZco8jhI2kZLhuuUWQoSIfhsuEaNQ4fSSPAWVQ6Uq31FJKcDnwCeB4wD1xTVR9a1OZ84DPAt/u7bqqqK9uqSRpFzqLS0Whz+GgOeGdV3ZnkJGA6yS1V9Y1F7W6vqotarEMaaUvNojIUtJzWho+q6oGqurO//T1gD3BqW68naWnOotLRWJUTzUm2A+cAdyxx98uS3A18B/jVqrp3icdfAVwBcMYZZ7RXqLQGOYtKRyNV1e4LJCcCtwG/WVU3Lbrv2cB8VT2W5DXAh6rqzMM93/j4eE1NTbVXsCStQUmmq2p8pXatzj5KsgH4NLB7cSAAVNWjVfVYf/tzwIYkW9qsSZK0vNZCIUmA64A9VfWBZdo8r9+OJOf163EtAUnqSJvnFF4OXAZ8Lcld/X3vAs4AqKqPAK8H3pxkDngcuKTaHs+SJC2rtVCoqq8AWaHNVcBVbdUgSTo6XtEsSWoYCpKkhqEgSWoYCpKkhqEgSWoYCpKkhqEgSWqsGApJ3prEFbQkaQQcSU/hecBfJLkhyYWHlqWQJK09K4ZCVf0GcCa9dYwuB76V5D8m+dGWa5MkrbIjOqfQX4/or/tfc8Am4MYkv9NibZKkVbbi2kdJ3ga8EXgIuBb4tap6Isk64FvAr7dboiRptRzJgnhbgJ+vqpmFO6tqPomfrSxJa8iKoVBV7z7MfXuObTmSpC55nYIkqWEoSJIahoIkqWEoSJIahoIkqWEoSJIahoIkqWEoSJIahoIkqWEoSJIahoIkqWEoSJIahoIkqWEoSJIahoKG2vTMLFffupfpmdmuS5HWhCP5kB1pIE3PzHLptZMcnJtn49g6du+aYOe2TV2XJQ01ewoaWpP7DnBwbp75gifm5pncd6DrkqShZyhoaE3s2MzGsXWsD2wYW8fEjs1dlyQNPYePNLR2btvE7l0TTO47wMSOzQ4dSceAoaChtnPbJsNAOoYcPpIkNVoLhSSnJ7k1yZ4k9yZ5+xJtkuT3kuxNck+Sc9uqR5K0sjaHj+aAd1bVnUlOAqaT3FJV31jQ5h8CZ/a/Xgp8uP9dktSB1noKVfVAVd3Z3/4esAc4dVGzi4FPVM8kcHKSU9qqSZJ0eKtyTiHJduAc4I5Fd50K3Lfg9v08NThIckWSqSRT+/fvb6tMSRp5rYdCkhOBTwPvqKpHF9+9xEPqKTuqrqmq8aoa37p1axtlSpJoORSSbKAXCLur6qYlmtwPnL7g9mnAd9qsSZK0vDZnHwW4DthTVR9YptnNwBv6s5AmgO9W1QNt1SRJOrw2Zx+9HLgM+FqSu/r73gWcAVBVHwE+B7wG2Av8P+BNLdYjSVpBa6FQVV9h6XMGC9sU8Ja2apAkHR2vaJYkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQyFATE9M8vVt+5lema261LUEX8GNAjGui5AvT8Gl147ycG5eTaOrWP3rgl2btvUdVlaRf4MaFDYUxgAk/sOcHBunvmCJ+bmmdx3oOuStMr8GdCgMBQGwMSOzWwcW8f6wIaxdUzs2Nx1SVpl/gxoUKSquq7hqIyPj9fU1FTXZRxz0zOzTO47wMSOzQ4bLDIqx2ZU3qe6kWS6qsZXauc5hQGxc9sm/xAsYZTG2v0Z0CBw+EgDzbF2aXUZChpojrVLq8vhIw20nds2sXvXhGPt0ioxFDTwHGuXVo/DR5KkhqEgl1eQ1HD4aMSN0pRPSSuzpzDinPIpaSFDYcQ55VPSQg4fjTinfEpaqLVQSPJR4CLgwao6a4n7zwc+A3y7v+umqrqyrXq0PKd8SjqkzZ7Cx4GrgE8cps3tVXVRizVIko5Ca+cUqurLwMNtPb8k6djr+kTzy5LcneTzSV68XKMkVySZSjK1f//+1axPkkZKl6FwJ7Ctql4C/Bfgvy3XsKquqarxqhrfunXrqhUoSaOms1Coqker6rH+9ueADUm2dFWPJKnDUEjyvCTpb5/Xr8UrpySpQ21OSf0kcD6wJcn9wHuADQBV9RHg9cCbk8wBjwOX1LB9NqgkrTGthUJV/eIK919Fb8qqJGlAdD37SJI0QAwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUFhF0zOzXH3rXqZnZrsuRZKW1ObHcWqB6ZlZLr12koNz82wcW8fuXRN+LrKkgWNPYZVM7jvAwbl55guemJtncp+rhEsaPIbCKpnYsZmNY+tYH9gwto6JHZu7LkmSnsLho1Wyc9smdu+aYHLfASZ2bHboSNJAMhRW0c5tmwwDSQPN4SNJUmNkQsHpoJK0spEYPnI6qCQdmZHoKTgdVJKOzEiEgtNBJenIjMTwkdNBJenIjEQogNNBJelIjMTwkSTpyBgKkqSGoSBJahgKkqSGoSBJahgKkqRGqqrrGo5Kkv3AzBE03QI81HI5w8jjsjyPzdI8LssbpmOzraq2rtRo6ELhSCWZqqrxrusYNB6X5XlsluZxWd5aPDYOH0mSGoaCJKmxlkPhmq4LGFAel+V5bJbmcVnemjs2a/acgiTp6K3lnoIk6SgZCpKkxpoKhSSnJ7k1yZ4k9yZ5e9c1DZIk65N8Nclnu65lkCQ5OcmNSb7Z/9l5Wdc1DYokv9L/Xfp6kk8mOa7rmrqS5KNJHkzy9QX7npPkliTf6n8f+vX511QoAHPAO6vqJ4EJ4C1JXtRxTYPk7cCerosYQB8C/qSqfgJ4CR4jAJKcCrwNGK+qs4D1wCXdVtWpjwMXLtr3b4A/raozgT/t3x5qayoUquqBqrqzv/09er/cp3Zb1WBIchrwj4Bru65lkCR5NvCzwHUAVXWwqh7ptqqBMgYcn2QMOAH4Tsf1dKaqvgw8vGj3xcD1/e3rgZ9b1aJasKZCYaEk24FzgDu6rWRg/C7w68B814UMmB3AfuBj/aG1a5M8q+uiBkFV/V/gPwN/BTwAfLeq/ke3VQ2cv1tVD0Dvn1LguR3X84ytyVBIciLwaeAdVfVo1/V0LclFwINVNd11LQNoDDgX+HBVnQN8nzUwBHAs9MfHLwZeADwfeFaSX+q2KrVtzYVCkg30AmF3Vd3UdT0D4uXAa5P8JfBHwAVJ/mu3JQ2M+4H7q+pQj/JGeiEheDXw7araX1VPADcBP9NxTYPmb5KcAtD//mDH9TxjayoUkoTe2PCeqvpA1/UMiqr6t1V1WlVtp3ei8ItV5X98QFX9NXBfkh/v73oV8I0OSxokfwVMJDmh/7v1KjwJv9jNwBv7228EPtNhLcfEWNcFHGMvBy4Dvpbkrv6+d1XV5zqsSYPvXwK7k2wE9gFv6riegVBVdyS5EbiT3sy+r7IGl3U4Ukk+CZwPbElyP/Ae4LeAG5L8Mr0Q/YXuKjw2XOZCktRYU8NHkqRnxlCQJDUMBUlSw1CQJDUMBUlSw1CQJDUMBUlSw1CQnqEkP53kniTHJXlW//MHzuq6Lunp8OI16RhI8h+A44Dj6a2l9P6OS5KeFkNBOgb6S2T8BfAD4Geq6smOS5KeFoePpGPjOcCJwEn0egzSULKnIB0DSW6mtyz5C4BTquqtHZckPS1rbZVUadUleQMwV1V/mGQ98D+TXFBVX+y6Nulo2VOQJDU8pyBJahgKkqSGoSBJahgKkqSGoSBJahgKkqSGoSBJavx/pnuMTLZ9apEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x28288f5f828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 1.创建数据\n",
    "x_train = np.array([3.3, 4.4, 5.5, 6.71, 6.93, 4.168, 9.779, 6.182, 7.59,\n",
    "                    2.167, 7.042, 10.791, 5.313, 7.997, 3.1], dtype=np.float32)\n",
    "y_train = np.array([1.7, 2.76, 2.09, 3.19, 1.694, 1.573, 3.366, 2.596, 2.53, 1.221,\n",
    "                   2.827, 3.465, 1.65, 2.904, 1.3], dtype=np.float32)\n",
    "x_train = np.reshape(x_train, [-1, 1])  # 转换为列向量\n",
    "y_train = np.reshape(y_train, [-1, 1])\n",
    "plt.plot(x_train, y_train, '.')\n",
    "plt.title('x-y raw data')\n",
    "plt.xlabel('x')\n",
    "plt.ylabel('y')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([15, 1])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将numpy的数据转换为torch的tensor\n",
    "x_train = torch.from_numpy(x_train)\n",
    "y_train = torch.from_numpy(y_train)\n",
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2.构建一维线性回归模型，即 y = w*x + b\n",
    "class LinearRegression(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()              # python3中继承类的初始化方法\n",
    "        self.linear = nn.Linear(1, 1)   # 定义一维的输入和输出\n",
    "    def forward(self, x):               # 模型的前向传播\n",
    "        output = self.linear(x)         # 计算output=w*x+b\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.一维线性回归模型:\n",
      " LinearRegression(\n",
      "  (linear): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# 3.实例化一个模型，并定义损失函数和优化函数\n",
    "# if torch.cuda.is_available():\n",
    "#     model = LinearRegression().cuda()   # 将模型放到GPU上执行\n",
    "# else:\n",
    "#     model = LinearRegression()\n",
    "model = LinearRegression().to(device)     # 如果GPU可用则优先选择\n",
    "print('1.一维线性回归模型:\\n',model)\n",
    "criterion = nn.MSELoss()                             # 使用均方误差损失\n",
    "optimizer = optim.SGD(model.parameters(), lr=1e-3)   # 使用SGD优化方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[20/300], loss: 1.671448\n",
      "Epoch[40/300], loss: 0.211000\n",
      "Epoch[60/300], loss: 0.171646\n",
      "Epoch[80/300], loss: 0.170569\n",
      "Epoch[100/300], loss: 0.170524\n",
      "Epoch[120/300], loss: 0.170507\n",
      "Epoch[140/300], loss: 0.170491\n",
      "Epoch[160/300], loss: 0.170475\n",
      "Epoch[180/300], loss: 0.170459\n",
      "Epoch[200/300], loss: 0.170443\n",
      "Epoch[220/300], loss: 0.170427\n",
      "Epoch[240/300], loss: 0.170412\n",
      "Epoch[260/300], loss: 0.170397\n",
      "Epoch[280/300], loss: 0.170381\n",
      "Epoch[300/300], loss: 0.170366\n"
     ]
    }
   ],
   "source": [
    "# 4.开始训练模型\n",
    "num_epochs = 300\n",
    "# 先将训练数据变为Variable,x_train包含所有的数据\n",
    "# inputs, target = (Variable(x_train).cuda(), Variable(y_train).cuda()) \\\n",
    "#                 if torch.cuda.is_available() \\\n",
    "#                 else (Variable(x_train), Variable(y_train))\n",
    "inputs, target = x_train.to(device), y_train.to(device)\n",
    "for epoch in range(num_epochs):\n",
    "    # forward\n",
    "    out = model(inputs)                # 先进行前向传播\n",
    "    loss = criterion(out, target)      # 计算损失函数\n",
    "    # backward\n",
    "    optimizer.zero_grad()              # 每次反向传播先归零梯度\n",
    "    loss.backward()                    # 反向传播\n",
    "    optimizer.step()                   # 更新参数\n",
    "    # validate\n",
    "    if (epoch + 1) % 20 == 0:\n",
    "        print('Epoch[{}/{}], loss: {:.6f}'.format(epoch+1, num_epochs, loss.data.item()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.FloatTensor\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VNX9//HXSVhCAAXFBYEwINSyBwxQ18qOrJaiqPlZsf2Kita9VoiCWyg/5evy/VK1QS0u0dYiWCuLKUUEWkTCJqsCEiBAZbEIIWwh5/vHhDF3mJAJmcm9M3k/Hw8ek3tyZubjRd45OffMucZai4iIxJcEtwsQEZHIU7iLiMQhhbuISBxSuIuIxCGFu4hIHFK4i4jEIYW7iEgcUriLiMQhhbuISByq4dYbN2rUyPp8PrfeXkQkJi1btmyvtfa88vq5Fu4+n4/c3Fy33l5EJCYZY7aG00/TMiIicUjhLiIShxTuIiJxyLU591COHz9Ofn4+R44ccbsUKSUpKYmmTZtSs2ZNt0sRkTCVG+7GmCRgAVC7pP80a+34oD4jgeeAHSVNk621r1W0mPz8fOrXr4/P58MYU9GnSxRYa9m3bx/5+fm0aNHC7XJEJEzhjNyPAj2ttQXGmJrAImPMbGvt50H9/mytvacyxRw5ckTB7jHGGM4991z27NnjdikiUgHlhrv136qpoOSwZsmfqN2+ScHuPfo7EYk9YV1QNcYkGmNWAruBv1trl4To9nNjzJfGmGnGmGYRrVJEJA4cOlrEpE++Yuf+w1F/r7DC3Vp7wlqbCjQFuhlj2gd1+Rvgs9Z2BOYCb4Z6HWPMKGNMrjEm16u/5ufn5zN06FBat27NxRdfzH333cexY8dC9t25cyfDhw8v9zUHDBjA/v37z6ieJ554gkmTJoVsb9KkCampqbRu3Zphw4axbt26cl9v6tSp7Ny584xqEZEz9+ycDbQb/wmTP93EoquHgM8H2dlRe78KLYW01u4H5gP9g9r3WWuPlhxOAS4t4/lZ1to0a23aeeeV++nZ8mVn+09QQkJETpS1lmHDhnHdddexceNGvv76awoKCsjIyDilb1FRERdddBHTpk0r93VnzZpFgwYNKlVbKA888AArV65k48aNjBgxgp49e5Y7N65wF6laK7fvx/foTF6evxmAW5f9jRtWz4WtW2HUqKgFfLnhbow5zxjToOTrOkBvYENQn8alDocA6yNZZEjZ2f4Ts3UrWBuREzVv3jySkpK47bbbAEhMTOSFF17gjTfeoLCwkKlTp3L99dczePBg+vbtS15eHu3b+3+JKSws5IYbbqBjx46MGDGC7t27B7ZX8Pl87N27l7y8PNq0acPtt99Ou3bt6Nu3L4cP+389mzJlCl27dqVTp078/Oc/p7CwsEK1jxgxgr59+/Luu+8C8NRTT9G1a1fat2/PqFGjsNYybdo0cnNzSU9PJzU1lcOHD4fsJyKVd/jYCbplzuW63/8TgMTiE6x6cQRPzv3DD50KCyHE4DESwhm5NwY+NcZ8CSzFP+f+sTHmKWPMkJI+9xpj1hpjVgH3AiOjUm1pGRn+E1NaJU/U2rVrufRS5y8dZ511FikpKWzatAmAxYsX8+abbzJv3jxHv5dffpmGDRvy5Zdf8vjjj7Ns2bKQ77Fx40buvvtu1q5dS4MGDfjggw8AGDZsGEuXLmXVqlW0adOG119/vcL1d+nShQ0b/D9377nnHpYuXcqaNWs4fPgwH3/8McOHDyctLY3s7GxWrlxJnTp1QvYTkcp5ce7XtBk3h90H/RMa7/yqO5snXcfZRw+d2nnbtqjUEM5qmS+BziHax5X6egwwJrKllaOsE1KJE2WtDbkypHR7nz59OOecc07ps2jRIu677z4A2rdvT8eOHUO+R4sWLUhNTQXg0ksvJS8vD4A1a9bw2GOPsX//fgoKCujXr98Z1X/Sp59+yrPPPkthYSHfffcd7dq1Y/Dgwac8J9x+IlK+NTu+Z9D/Lgoc39StGb8bVpIFKSn+GYZgKSlRqSV2tx8o64RU4kS1a9fulJ0qDxw4wPbt27n44osBqFu3bsjnhjudUbt27cDXiYmJFBUVATBy5EgmT57M6tWrGT9+/Bl9SnfFihW0adOGI0eOMHr0aKZNm8bq1au5/fbbQ75euP1E5PSOHD/BVc/OcwT7isf7/BDsAJmZkJzsfGJysr89CmI33KNwonr16kVhYSFvvfUWACdOnOChhx5i5MiRJAe/V5Arr7yS999/H4B169axevXqCr33wYMHady4McePHyf7DK4bfPDBB+Tk5HDTTTcFArpRo0YUFBQ4LvrWr1+fgwcPApy2n4iE55X5m/nx43PY/p3/+tkfb+tK3sSBNKxby9kxPR2ysqB5czDG/5iV5W+PAk/tLVMhJ09IRoZ/KiYlxR/slThRxhhmzJjB6NGjefrppykuLmbAgAFMmDCh3OeOHj2aW2+9lY4dO9K5c2c6duzI2WefHfZ7P/3003Tv3p3mzZvToUOHQACfzgsvvMA777zDoUOHaN++PfPmzePkKqTbb7+dDh064PP56Nq1a+A5I0eO5M4776ROnTosXry4zH4icnob/n2A/i8uDBwP69yE/76h0+k/9JeeHrUwD2bcWh2RlpZmg6dA1q9fT5s2bVypp7JOnDjB8ePHSUpKYvPmzfTq1Yuvv/6aWrVqlf/kGBDLfzdSjWVnR3QACHCsqJhrX1rA5j0/XBzNfaw3jerVPs2zIscYs8xam1Zev9gduXtMYWEhPXr04Pjx41hreeWVV+Im2EVi0snl0idX1Z1cLg1nHPCvLfyGZ2b+sNJ7yi/S6NP2gspWGhUK9wipX7++bhso4iWnWy5dwXDftPsgvZ9fEDge2KExk2/u7Ol9lxTuIhKfIrBc+viJYoZO/ifrdh0ItH2R0Yvz6ydVtrqoU7iLSHyq5Lrytxfn8fhf1waOX07vwoAOjct+gsco3EUkPmVmOufcIazl0lv2HqLHpPmB495tzmfKL9I8PQUTisJdROJTBZdLF50o5vo/LGbFth92cF08pieNz65TFdVGXOx+iClKEhMTSU1NDfzJy8sjNzeXe++9F4D58+fzr3/9K9D/ww8/dGy1O27cOObOnRuRWk5uOFbaRx99xMSJEyPy+iJxLz0d8vKguNj/WEaw/+mLbbTKmB0I9pduTCVv4sCYDXbQyP0UderUYeXKlY42n89HWpp/Wen8+fOpV68el19+OeAP90GDBtG2bVvAvxtjNA0ZMoQhQ4aU31FEyrX9u0KuevbTwPFVrRvx5m3dSEiIrSmYUDRyD8P8+fMZNGgQeXl5vPrqq7zwwgukpqby2Wef8dFHH/Gb3/yG1NRUNm/ezMiRIwMf4/f5fIwfP54uXbrQoUOHwI6Ne/bsoU+fPnTp0oU77riD5s2bnzJCL8vUqVO55x7/rWpHjhzJvffey+WXX07Lli0d2wc899xzdO3alY4dOzJ+/PiyXk6kWjpRbBnxh8WOYF/02x68/avucRHs4OGR+5N/W8u6nQfK71gBbS86i/GD2522z+HDhwO7NrZo0YIZM2YEvufz+bjzzjupV68eDz/8MOAfSQ8aNKjMOzI1atSI5cuX8/LLLzNp0iRee+01nnzySXr27MmYMWOYM2cOWVlZZ/zftGvXLhYtWsSGDRsYMmQIw4cPJycnh40bN/LFF19grWXIkCEsWLCAq6+++ozfRyReTF+ez4PvrwocPze8I9enxd+dQT0b7m4JNS1TGcOGDQP82/tOnz4d8G8PfPKHRv/+/WnYsOEZv/51111HQkICbdu25dtvvwUgJyeHnJwcOnf279RcUFDAxo0bFe5Sre3Yf5grJv5wH4ZuLc7hvdt/QmKcjNSDeTbcyxthx4qTW/yW3t43kvv5lN5C+OTrWmsZM2YMd9xxR8TeRyRWFRdbbpu6lM++/uEWlJ/95hqanxt6++54oTn3Ciq9ZW6o43CU3h44JyeH//znPxGtsV+/frzxxhsUFBQAsGPHDnbv3h3R9xCJBXe8nUvLsbMCwT7hZx3Imzgw7oMdFO4VNnjwYGbMmEFqaioLFy7kxhtv5LnnnqNz585s3rw5rNcYP348OTk5dOnShdmzZ9O4cWPq168fsm/Hjh1p2rQpTZs25cEHHwzr9fv27cvNN9/MZZddRocOHRg+fHiFfwCJxLIv8/03pf5k7beBtk2Z13Jz9+jc9ciLtOWvC44ePUpiYiI1atRg8eLF3HXXXRGd54+G6vJ3I7GtuNjScuwsR9trv0ijt0d3bjwT2vLXw7Zt28YNN9xAcXExtWrVYsqUKW6XJOGKwv7gEhkP/nkl01fsCBy3bFSXeQ9f415BLlO4u6B169asWLHC7TKkoqKwP7hU3vpdB7j2pYXOtqf6U6dWoksVeYPnwt1aG3Mb9MQ7t6buPCeC+4NL5VlraTHGOQUTazs3RpOnwj0pKYl9+/Zx7rnnKuA9wlrLvn37SEry/v7VUReB/cElMjJmrCZ7yQ/n/bz6tVma0dvFirzHU+HetGlT8vPz2bNnT/mdpcokJSXRtGlTt8twXyX3B5fK27S7gN7Pf+ZoW/NkP+rV9lSUeYKnzkjNmjVp0aKF22WIhHaG+4NL5YWagnn+hk4M66JBR1k8Fe4inlbB/cElMjJnrmPKwi2B4+Raiax7qr+LFcUGhbtIRaSnK8yryNZ9h/jpc/MdbavG9eXs5JruFBRjFO4i4jm+R2c6jn83rAM3ddO1jYpQuIuIZzyf8xX/M2+Toy1v4kCXqoltCncRcV3wdrwAyx7rzbn1apfxDCmPwl1EXBU8BTNuUFt+eaVWzVWWwl1EXPHy/E08O+crR5umYCJH4S4iVWr3gSN0m/APR9sXY3tx/ln6FHQkKdxFpMoET8E80v8SRl/TyqVq4pvCXUSi7o1FW3jq43WONk3BRFe54W6MSQIWALVL+k+z1o4P6lMbeAu4FNgHjLDW5kW8WhGJKfsKjnLpM3Mdbf96tCcXNajjUkXVRzgj96NAT2ttgTGmJrDIGDPbWvt5qT6/Av5jrW1ljLkR+P/AiCjUKyIxIngK5t6erXiw7yUuVVP9lBvu1r+Zd0HJYc2SP8EbfA8Fnij5ehow2RhjrDYCF6l23l2yjbEzVjvaNAVT9cKaczfGJALLgFbA7621S4K6NAG2A1hri4wx3wPnAnuDXmcUMAogRdukisSV7wuP0+mpHEfbgt/0IOXcZJcqqt7CCndr7Qkg1RjTAJhhjGlvrV1TqkuoO2ucMmq31mYBWeC/QfYZ1CsiHtTm8TkcPn4icPxfV7bgsUFtXaxIKrRaxlq73xgzH+gPlA73fKAZkG+MqQGcDXwXqSJFxJumL8/nwfdXOdq2/G6A7qTmAeGsljkPOF4S7HWA3vgvmJb2EXArsBgYDszTfLtI/Co4WkT78Z842uY++FNanV/PpYokWDgj98bAmyXz7gnA+9baj40xTwG51tqPgNeBt40xm/CP2G+MWsUi4qqumXPZc/Bo4Di9ewqZP+vgYkUSSjirZb4EOodoH1fq6yPA9ZEtTUS8ZOaXu7j73eWONk3BeFeC2wWISJRlZ4PPBwkJ/sfs7Ao9vfBYEb5HZzqCfc79V5E3caCC3cO0/YBIPMvOdt7Ue+tW/zGEdbvAnpPm883eQ4Hjn3VuwgsjUqNRqUSYceu6Z1pams3NzXXlvUWqDZ/PH+jBmjeHvLwynzZ33bf811vOf5/fTBhAQoJG6m4zxiyz1qaV108jd5F4tm1bhdqPHD/Bjx+f42j76J4r6Ni0QaQrkyhTuIvEs5SU0CP3EJ8QH/S/C1mz40DguH+7C3n1lkujWZ1EkcJdJJ5lZjrn3AGSk/3tJRZu3MMtr3/heNrmCQNI1BRMTNNqGZFoqeQqlYhIT4esLP8cuzH+x6wsSE/nWFExvkdnOoL9g7suI2/iQAV7HNDIXSQaKrlKJaLS0095zxF/WMySLT/sEHJV60a8/avuVVuXRJVWy4hEwxmuUom2Jd/sY0TW5462jZnXUjNRv8THCq2WEXFTBVepRFvRiWJaZcx2tL37X925vFUjV+qR6FO4i0RDBVapRFvwHZG6pDRg+ugrqrwOqVoKd5FoCGOVSrTNXr2Lu7Kde8F89Ux/atdIrLIaxD2aaKsuvLByozo5zSqVaCs64V8FUzrYnxzSjryJAxXs1YhG7tWBl1ZuVCchVqlEW8sxMykOWiOh+5dWT1otUx14dOWGRM6nX+3mtj8udbStebIf9Wpr/BZvtFpGfuCxlRsSOcXFlpZjZznaHul/CaOvaeVSReIVCvfqwEMrNyRyOj+Vw38KjzvaNAUjJyncqwMPrNyQyFm8eR83TXF+EGnluD40SK7lUkXiRQr36uDkRb2MDP9UTEqKP9h1MTWmWGtpMcY5BXNPj1Y83O8SlyoSL1O4VxcurNyQyAn+IBJoCkZOT+Eu4mHzNnzLL6c6V5X989GeNGlQx6WKJFYo3EU8KNQUTFdfQ/5y5+UuVSSxRuEu4jGagpFIULiLeESoVTCfPnwNLRrVdakiiWUKdxEPCB6t/+iCeuQ88FOXqpF4oHAXcVGrsbMoCtoMRlMwEgnaFVLinwd3xFy5fT++R2c6gn3O/Vcp2CViNHKX+ObBHTGDp2DOq1+bpRm9XalF4pd2hZT45qEdMdOemcvegqOONo3UpaK0K6QIeGJHzPW7DnDtSwsdbR/efQWpzRpUWQ1S/SjcJb65vCNm8BRMzUTDxswBVfLeUr0p3CW+ubQjZp/nP2Pj7gJHm6ZgpCpptYzEtyq+l+m6nQfwPTrTEex/GvWTyAS7B1f9iHdp5C7xr4p2xIzqtgEeXPUj3lbuahljTDPgLeBCoBjIsta+FNTnGuCvwJaSpunW2qdO97paLSPxou24ORQeO+Fo2/K7ARhjIvcmHlr1I+6K5GqZIuAha+1yY0x9YJkx5u/W2nVB/RZaawedSbEisWjL3kP0mDTf0fbCiE78rHPTyL+ZB1b9SGwpN9yttbuAXSVfHzTGrAeaAMHhLlJtVPnOjboPrlRQhebcjTE+oDOwJMS3LzPGrAJ2Ag9ba9dWujoRj7nq2Xls/+6woy3iUzCh6D64UkFhh7sxph7wAXC/tfZA0LeXA82ttQXGmAHAh0DrEK8xChgFkKIRh8SQHfsPc8XEeY62p4e245bLfFVTgO6DKxUU1vYDxpiawMfAJ9ba58PonwekWWv3ltVHF1QlVujmGeIlEbugavy/b74OrC8r2I0xFwLfWmutMaYb/vXz+ypYs4inDP39P1m1fb+jbfOEASQmRHkKRiQCwpmWuQK4BVhtjFlZ0jYWSAGw1r4KDAfuMsYUAYeBG61bO5KJVNLegqOkPTPX0fZI/0sYfU0rlyoSqbhwVsssAk47VLHWTgYmR6ooEbdoCkbihT6hKgKM/OMXzP9qj6NtY+a11EzUDh0SmxTuUq19f/g4nZ7McbTd8dOWjLm2jUsViUSGwl2qLU3BSDxTuEu1c/+fVvDhyp2Otg1P9yepZqJLFYlEnsJdqo3CY0W0HfeJo+2mbin8blgHlyoSiR6Fu1QLmoKR6kbhLnEta8FmJsza4Ghb82Q/6tXW//oS3/R/uMSlY0XF/Oix2Y42rYKR6kThLnFHUzAiCneJI9lLtpIxY42j7csn+nJWUk2XKhJxj8JdYt6JYsvFY2c52m7unsKEn2kVjFRfCneJaZqCEQlN4S4x6cMVO7j/zysdbcsf78M5dWu5VJGItyjcJaYUF1taBk3BDOrYmMk3d3GpIhFvUrhLzNAUjEj4FO7ieTlr/82ot5c52paM7cUFZyW5VJGI9yncxbOstbQY45yCufpH5/HWL7u5VJFI7FC4iydpCkakchTu4ikLN+7hlte/cLY90oNm5yS7VJFIbFK4i2cEj9Y7NT2bv95zpUvViMQ2hbu4TlMwIpGncBfX5OZ9x/BXFzva5j54Na3Or+9SRSLxQ+Eurggeraeck8yCR3q4VI1I/FG4S5Xq8MQnHDxS5GjTFIxI5CncpUps2XuIHpPmO9o+/vWVtG9ytjsFicQ5hbtEXfAUTP2kGqx+op9L1YhUDwp3iZp731vBR6t2Oto0BSNSNRLcLkDiT/5/CvE9OtMR7J/cf3XZwZ6dDT4fJCT4H7Ozq6ROkXimkbtEVPAUTFdfQ/5y5+VlPyE7G0aNgsJC//HWrf5jgPT0KFUpEv+MtdaVN05LS7O5ubmuvLdE3tgZq3l3yTZHW1hTMD6fP9CDNW8OeXkRqU0knhhjlllr08rrp5G7VMruA0foNuEfjra/3XMlHZqGuQpm27aKtYtIWBTucsaCp2AuuaA+nzxwdcVeJCUl9Mg9JaUSlYmIwl0qbMKs9WQt+MbRtuV3AzDGVPzFMjOdc+4Aycn+dhE5Ywp3Cdt3h47R5em/O9r+cudldPWdc+YvevKiaUaGfyomJcUf7LqYKlIpCncJS/AUTOOzk1g8pldkXjw9XWEuEmHlhrsxphnwFnAhUAxkWWtfCupjgJeAAUAhMNJauzzy5UpVe3Hu17w4d6Oj7YynYESkyoQzci8CHrLWLjfG1AeWGWP+bq1dV6rPtUDrkj/dgVdKHiVGHThynI5P5Dja3vlVd65s3cilikSkIsoNd2vtLmBXydcHjTHrgSZA6XAfCrxl/YvmPzfGNDDGNC55rsSY4CmY5FqJrHuqv0vViMiZqNCcuzHGB3QGlgR9qwmwvdRxfkmbI9yNMaOAUQApWurmOVMWfEPmrPWOts0TBpCYoCkYkVgTdrgbY+oBHwD3W2sPBH87xFNO+eirtTYLyAL/J1QrUKdE0aGjRbQb/4mjbcov0ujT9gKXKhKRygor3I0xNfEHe7a1dnqILvlAs1LHTYGdIfqJx+j+pSLxKZzVMgZ4HVhvrX2+jG4fAfcYY/6E/0Lq95pv97bsJVvJmLHG0bYp81pqJGqjUJF4EM7I/QrgFmC1MWZlSdtYIAXAWvsqMAv/MshN+JdC3hb5UiUSjhw/wY8fn+No+5+bOjOk00UuVSQi0RDOaplFhJ5TL93HAndHqiiJDk3BiFQf+oRqNTBjRT4P/HmVo+2rZ/pTu0aiSxWJSLQp3OPYsaJifvTYbEfbsz/vyA1dm5XxDBGJFwr3OKUpGJHqTeEeZ2av3sVd2c5tfdY/1Z86tTQFI1KdKNzjRHGxpeXYWY62Jwa3ZeQVLVyqSETcpHCPA/e8u5yPv3R+rEBTMCLVm8I9hq3Z8T2D/neRo23D0/1JqqkpGJHqTuEeg6y1tBjjnIL5wy2X0q/dhS5VJCJeo3CPMY9MW8X7ufmB46YN67Dotz1drEhEvEjhHiO+/vYgfV9Y4Ghb+2Q/6tbWX6GInErJ4HGhpmC0F4yIlEfh7mHj/7qGNxdvDRw3SK7JynF9XaxIRGKFwt2DvtlTQM///szR9uUTfTkrqaZLFYlIrFG4e0ioKZhnh3fkhjTtBSMiFaNw94iJszfw6mebA8c1EgybJgxwsSIRiWUKd5dt/66Qq5791NG24vE+NKxby6WKRCQeKNxdFLxz49ND23HLZT53ihGRuKIbZkZSdjb4fJCQ4H/Mzg7Z7cW5X58S7HkTByrYRSRiFO6Rkp0No0bB1q1grf9x1ChHwP/7+yP4Hp3Ji3M3BtqWZvTWJl/xJMwf8CLRZvy3P616aWlpNjc315X3jgqfzx/owZo3h7y8U0bqGQPacPvVLaumNqkaJ3/AFxb+0JacDFlZkJ7uXl0SV4wxy6y1aeX108g9UrZtC9mcdcGlIadgFOxxKCPDGezgP87IcKceqdZ0QTVSUlIcI/c9yQ3o+ut3HF0+H9OLC89OqurKpKqU8QO+zHaRKNLIPVIyM/2/ggO+337sCPYH+/yIvIkDq3ewV4e56JSUirWLRJFG7pGSns4n39fkjm11Hc26WMqpc9EnLzZDfM1FZ2aGnnPPzHSvJqm2dEE1Ag4eOU6HJ3IcbQsf6UGzc5JdqshjyrnYHFeys/1z7Nu2+UfsmZnx9QNMXBfuBVWFeyXdPOVz/rV5X+B4ws86cHN3/RrukJDgXx4azBgoLq76ekRiWLjhrmmZM/TpV7u57Y9LA8eN6tUm97HeLlbkYUEXmx3tIhIVCvcKOnL8BD9+fI6jbfGYnjQ+u45LFcUAzUWLVDmtlqmAV+ZvdgT7+MFtyZs40BvB7uXVKOnp/g/yNG/un4pp3lwf7BGJMo3cw7Dh3wfo/+LCwPGwzk347xs6YYxxsapSYmE1Snq6d2oRqQZ0QfU0jhUVc+1LC9i851CgLfex3jSqV9vFqkKoTqtRRKo5XVCtpNcWfsMzM9cHjqf8Io0+bS9wsaLT0CcjRSSIwj3Ipt0H6f38gsDxwA6NmXxzZ+9MwYSi1SgiEkThXuL4iWKGTv4n63YdCLR9kdGL8+vHwJYBWo0iIkHKXS1jjHnDGLPbGLOmjO9fY4z53hizsuTPuMiXGV1vL86jdcbsQLC/nN6FvIkDYyPYQatRROQU4YzcpwKTgbdO02ehtXZQRCqqQlv2HqLHpPmB495tzmfKL9K8PQVTFq1GEZFSyg13a+0CY4wv+qVUnaITxVz/h8Ws2LY/0KYPIolIPInUnPtlxphVwE7gYWvt2gi9bsT96YttPDp9deD4pRtTGZraxMWKREQiLxLhvhxobq0tMMYMAD4EWofqaIwZBYwCSKnilRzbvyvkqmc/DRxf1boRb97WjYSEGJyCEREpR6XD3Vp7oNTXs4wxLxtjGllr94bomwVkgf9DTJV973CcKLbcPOVzlmz5LtC26Lc9aNpQ2/GKSPyqdLgbYy4EvrXWWmNMN/wrcPaV87QqMX15Pg++vypw/Nzwjlyf1szFikREqka54W6MeQ+4BmhkjMkHxgM1Aay1rwLDgbuMMUXAYeBG69aeBiV27D/MFRPnBY67tTiH927/CYmaghGRaiKc1TI3lfP9yfiXSrquuNhy29SlfPb1nkDbZ7+5hubn1j3Ns0RE4k/cfEL1b6t28uv3VgSOdUckEanOYj7cvz1whO4T/hE47tT0bD6463JqJGqrehGpvmI23K21jHp7GX9f922gbd5DP6XlefVcrEpExBtiMtznrPk3d76J/HqEAAAEZUlEQVSzLHD8xOC2jLyihYsViYh4S8yF+6bdBYFg//GF9fnbr6+kpqZgREQcYi7cL2qQxC+vaMFN3ZrR+oL6bpcjIuJJMRfuybVqMG5wW7fLEBHxNM1niIjEIYW7iEgcUrhXVHY2+HyQkOB/zM52uyIRkVPE3Jy7q7Kznfcq3brVfwy6C5KIeIpG7hWRkeG8CTX4jzMy3KlHRKQMCveK2LatYu0iIi5RuFdEWXePquK7SomIlEfhXhGZmZAcdAen5GR/u4iIhyjcKyI9HbKyoHlzMMb/mJWli6ki4jmxFe5eWIaYng55eVBc7H9UsIuIB8XOUkgtQxQRCVvsjNy1DFFEJGyxE+5ahigiErbYCXctQxQRCVvshLuWIYqIhC12wl3LEEVEwhY7q2XAH+QKcxGRcsXOyF1ERMKmcBcRiUMKdxGROKRwFxGJQwp3EZE4ZKy17ryxMXuArWF0bQTsjXI5sUjnpWw6N6HpvJQtls5Nc2vteeV1ci3cw2WMybXWprldh9fovJRN5yY0nZeyxeO50bSMiEgcUriLiMShWAj3LLcL8Cidl7Lp3ISm81K2uDs3np9zFxGRiouFkbuIiFSQJ8PdGNPMGPOpMWa9MWatMeY+t2vyEmNMojFmhTHmY7dr8RJjTANjzDRjzIaS/3cuc7smrzDGPFDyb2mNMeY9Y0yS2zW5xRjzhjFmtzFmTam2c4wxfzfGbCx5bOhmjZHgyXAHioCHrLVtgJ8Adxtj2rpck5fcB6x3uwgPegmYY639MdAJnSMAjDFNgHuBNGtteyARuNHdqlw1Fegf1PYo8A9rbWvgHyXHMc2T4W6t3WWtXV7y9UH8/0ibuFuVNxhjmgIDgdfcrsVLjDFnAVcDrwNYa49Za/e7W5Wn1ADqGGNqAMnATpfrcY21dgHwXVDzUODNkq/fBK6r0qKiwJPhXpoxxgd0Bpa4W4lnvAg8AhS7XYjHtAT2AH8smbJ6zRhT1+2ivMBauwOYBGwDdgHfW2tz3K3Kcy6w1u4C/+ASON/leirN0+FujKkHfADcb6094HY9bjPGDAJ2W2uXuV2LB9UAugCvWGs7A4eIg1+tI6Fk/ngo0AK4CKhrjPl/7lYl0ebZcDfG1MQf7NnW2ulu1+MRVwBDjDF5wJ+AnsaYd9wtyTPygXxr7cnf8KbhD3uB3sAWa+0ea+1xYDpwucs1ec23xpjGACWPu12up9I8Ge7GGIN/7nS9tfZ5t+vxCmvtGGttU2utD/8FsXnWWo3AAGvtv4HtxphLSpp6AetcLMlLtgE/McYkl/zb6oUuNgf7CLi15Otbgb+6WEtEePUeqlcAtwCrjTErS9rGWmtnuViTeN+vgWxjTC3gG+A2l+vxBGvtEmPMNGA5/pVoK4jDT2SGyxjzHnAN0MgYkw+MByYC7xtjfoX/h+H17lUYGfqEqohIHPLktIyIiFSOwl1EJA4p3EVE4pDCXUQkDincRUTikMJdRCQOKdxFROKQwl1EJA79HyvenmByY6blAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x28288f5f3c8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 5.评估模型性能\n",
    "# %pylab    \n",
    "model.eval()                     # 模型变成测试模式\n",
    "model.cpu()                      # 将模型和测试数据都放到cpu上\n",
    "inputs = inputs.cpu()           \n",
    "# inputs = inputs.to(torch.device('cpu'))\n",
    "print(inputs.type())\n",
    "predict = model(inputs)          # 对输入数据的输出进行预测\n",
    "predict = predict.data.numpy()   # 转换为numpy数据才可以画图\n",
    "plt.plot(x_train.numpy(), y_train.numpy(), 'ro', label='Original Data')\n",
    "# 是一条直线，所以顺序不会影响绘图\n",
    "plt.plot(x_train.numpy(), predict, label='Fitting Line')  \n",
    "plt.legend(['Original Data', 'Fitting Line'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "小结:\n",
    "1. 创建数据或数据的读取及预处理\n",
    "2. 构建网络模型\n",
    "3. 实例化模型并定义损失函数及优化方法\n",
    "4. 读入数据训练网络，同时输出准确率的验证结果\n",
    "5. 测试模型的性能并可视化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 多项式回归\n",
    "每一维特征属性的次数不唯一"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.拼接后的矩阵:\n",
      " tensor([[  1.,   1.,   1.],\n",
      "        [  2.,   4.,   8.],\n",
      "        [  3.,   9.,  27.],\n",
      "        [  4.,  16.,  64.],\n",
      "        [  5.,  25., 125.],\n",
      "        [  6.,  36., 216.],\n",
      "        [  7.,  49., 343.]])\n",
      "2.满足多项式的输出:\n",
      " tensor([[  6.8000],\n",
      "        [ 33.1000],\n",
      "        [ 94.2000],\n",
      "        [204.5000],\n",
      "        [378.4000],\n",
      "        [630.3000],\n",
      "        [974.6000]])\n"
     ]
    }
   ],
   "source": [
    "# 1.数据的创建和预处理\n",
    "def make_features(x):\n",
    "    \"\"\"使数据X处理为多项式矩阵[x, x^2, x^3]\"\"\"\n",
    "    x =x.unsqueeze(1)            # 参数1变为列向量，参数0变为行向量\n",
    "    # 拼接的是tensor组成的列表，1表示按列拼接，0表示按行拼接 (0行1列)\n",
    "    return torch.cat([x**i for i in range(1,4)], 1)   # 按列拼接为矩阵\n",
    "# 使数据满足y=0.9+0.5*x+3*x^2+2.4*x^3关系，向量化为y = X*W + b\n",
    "w_target = torch.Tensor([0.5, 3, 2.4]).unsqueeze(1)   # 权重变为列向量\n",
    "b_target = torch.Tensor([0.9])\n",
    "def func(x):    # 构建函数关系式\n",
    "    return x.mm(w_target) + b_target    # y = X*W + b，得到列向量\n",
    "\n",
    "test = torch.Tensor([1,2,3,4,5,6,7])\n",
    "test_out = make_features(test)\n",
    "print('1.拼接后的矩阵:\\n', test_out)\n",
    "out = func(test_out)\n",
    "print('2.满足多项式的输出:\\n', out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 上述函数合并为一个process_data\n",
    "def process_data(input_x):           # 预处理数据\n",
    "    input_x = input_x.unsqueeze(1)   # 得到列向量\n",
    "    concat_x = torch.cat([input_x**i for i in range(1,4)], 1)  # 按列拼接为矩阵\n",
    "    w_target = torch.Tensor([0.5, 3, 2.4]).unsqueeze(1)        # 权重变为列向量\n",
    "    b_target = torch.Tensor([0.9])\n",
    "    output_y = concat_x.mm(w_target) + b_target                # 使数据符合多项式\n",
    "    return concat_x, output_y\n",
    "# test = torch.Tensor([1,2,3,4,5,6,7])\n",
    "# x, y = process_data(test)\n",
    "def get_batch(batch_size=32):      # 返回batch_size的(x,y)数据对用于训练\n",
    "    random_data = torch.randn(batch_size)            # 随机产生高斯分布的数据\n",
    "    x_train, y_train = process_data(random_data)     # 产生符合要求的数据\n",
    "#     inputs, target = (Variable(x_train).cuda(), Variable(y_train).cuda()) \\\n",
    "#                 if torch.cuda.is_available() \\\n",
    "#                 else (Variable(x_train), Variable(y_train))\n",
    "    inputs, target = x_train.to(device), y_train.to(device)\n",
    "    return inputs, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2.构建模型并定义损失函数及优化方法\n",
    "# 定义模型\n",
    "class poly_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.poly = nn.Linear(3,1)   # y = X*W + b, x为3维，y为1维\n",
    "    def forward(self, x):\n",
    "        return self.poly(x)\n",
    "# model = poly_model().cuda() if torch.cuda.is_available() else poly_model()\n",
    "model = poly_model().to(device)\n",
    "# 定义损失函数与优化方法\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num_batches[101], loss: 11.906305\n",
      "Num_batches[201], loss: 1.280829\n",
      "Num_batches[301], loss: 0.346025\n",
      "Num_batches[401], loss: 0.077029\n",
      "Num_batches[501], loss: 0.137702\n",
      "Num_batches[601], loss: 0.021907\n",
      "Num_batches[701], loss: 0.019165\n",
      "Num_batches[801], loss: 0.013940\n",
      "Num_batches[901], loss: 0.014522\n",
      "Num_batches[1001], loss: 0.027717\n",
      "Num_batches[1101], loss: 0.010648\n",
      "Num_batches[1201], loss: 0.007721\n",
      "Num_batches[1301], loss: 0.006071\n",
      "Num_batches[1401], loss: 0.005099\n",
      "Num_batches[1501], loss: 0.004672\n",
      "Num_batches[1601], loss: 0.004756\n",
      "Num_batches[1701], loss: 0.003049\n",
      "Num_batches[1801], loss: 0.001870\n",
      "Num_batches[1901], loss: 0.002457\n",
      "Num_batches[2001], loss: 0.001583\n"
     ]
    }
   ],
   "source": [
    "# 3.训练模型\n",
    "num_batches = 0       # ！！！注意这里的batch数据迭代的次数，不是epoch \n",
    "while True:\n",
    "    batch_x, batch_y = get_batch()       # 获取数据\n",
    "    output = model(batch_x)              # 前向传播\n",
    "    loss = criterion(output, batch_y)    # 计算损失函数\n",
    "    print_loss = loss.data.item()\n",
    "    optimizer.zero_grad()                # 梯度归零\n",
    "    loss.backward()                      # 误差反向传播\n",
    "    optimizer.step()                     # 更新参数\n",
    "    num_batches += 1        \n",
    "    if print_loss < 1e-3:\n",
    "        break\n",
    "    if num_batches % 100 == 0:\n",
    "        print('Num_batches[{}], loss: {:.6f}'.format(num_batches+1, print_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8FPX9x/HXhwAJASqnFbmCFCunEYNWPKoiaD2AIgpIK3gQAVFatT8PVNTWFlBLxQMKSMUST1DrgRYVKeIJKsplBZRAgGIAUULClXx/f8wmbEKODdns7G7ez8dj3M3sd2c+O8FPvvud73zGnHOIiEj8qOV3ACIiEl5K7CIicUaJXUQkziixi4jEGSV2EZE4o8QuIhJnlNhFROKMEruISJxRYhcRiTO1/dhps2bNXEpKih+7FhGJWZ9++ul251zzitr5kthTUlJYtmyZH7sWEYlZZpYZSjsNxYiIxBkldhGROKPELiISZ3wZYy/NgQMHyMrKYu/evX6HIgFJSUm0atWKOnXq+B2KiFRCyIndzFoDTwHHAAXAdOfcw2Z2DzACyA40vcM5N7+ygWRlZdGwYUNSUlIws8q+XcLMOceOHTvIysqiXbt2focjIpVQmaGYg8DNzrmOwC+A682sU+C1yc651MBS6aQOsHfvXpo2baqkHiXMjKZNm+oblEi4ZGRASgrUquU9ZmRU265C7rE757YCWwPPd5vZGqBlOINRUo8u+n2IhElGBqSnQ26u93NmpvczwNChYd/dEZ08NbMU4CTg48CqMWb2pZnNMrPGZbwn3cyWmdmy7Ozs0pqIiMSncePIzXWM5W9soK23LjcXxo2rlt1VOrGbWQNgHvA759yPwFSgPZCK16N/qLT3OeemO+fSnHNpzZtXeOGUL7KysujXrx8dOnSgffv2jB07lv3795fadsuWLQwcOLDCbV544YXs2rXriOK55557ePDBB0td37JlS1JTU+nQoQMDBgxg9erVFW7vySefZMuWLUcUi4hUwcaNzOJqpjCWTbQutr46VCqxm1kdvKSe4Zx7EcA5t805l++cKwBmAKeEP8xShHm8yjnHgAED6N+/P2vXruXrr78mJyeHcaX8RT148CDHHnssc+fOrXC78+fPp1GjRlWKrTS///3vWb58OWvXrmXQoEGce+65VPRNSIldxB8HWh/Hg9xCT97nDJYceqFNm2rZX8iJ3bwB1yeANc65vwatbxHU7NfAyvCFV4bC8arMTHDu0HhVFZL7woULSUpK4qqrrgIgISGByZMnM2vWLHJzc3nyySe57LLLuOSSS+jTpw8bNmygS5cuAOTm5nL55ZfTrVs3Bg0axKmnnlpUMiElJYXt27ezYcMGOnbsyIgRI+jcuTN9+vQhLy8PgBkzZtCjRw9OPPFELr30UnILx+FCNGjQIPr06cPTTz8NwH333UePHj3o0qUL6enpOOeYO3cuy5YtY+jQoaSmppKXl1dqOxEJv+cvmk0mKdzKRIrOXCUnw/33V8v+KtNjPx34LXCumS0PLBcCk8xshZl9CZwD/L46Ai1m3LhDJyEKVXG8atWqVZx88snF1v3kJz+hTZs2rFu3DoAPP/yQ2bNns3DhwmLtHn/8cRo3bsyXX37JXXfdxaefflrqPtauXcv111/PqlWraNSoEfPmzQNgwIABLF26lC+++IKOHTvyxBNPVDr+7t2789VXXwEwZswYli5dysqVK8nLy+O1115j4MCBpKWlkZGRwfLly6lXr16p7UQkvJyDiUtOp1PLXVzcZgWYQdu2MH16tZw4hcrNilkClDZN4oimN1ZJWeNSVRivcs6VOgskeH3v3r1p0qTJYW2WLFnC2LFjAejSpQvdunUrdR/t2rUjNTUVgJNPPpkNGzYAsHLlSu6880527dpFTk4O559//hHFX+jdd99l0qRJ5ObmsnPnTjp37swll1xy2HtCbSciR+6NN2DFCnjyyUbUGvZtRPYZmyUFyhqXqsJ4VefOnQ+rOPnjjz+yadMm2rdvD0D9+vVLfW+oQxiJiYlFzxMSEjh48CAAw4cP59FHH2XFihWMHz/+iOaOf/7553Ts2JG9e/cyevRo5s6dy4oVKxgxYkSp2wu1nYhUzcSJ0Lo1DBkSuX3GZmK//35vfCpYFcerevXqRW5uLk899RQA+fn53HzzzQwfPpzkkvsq4YwzzuD5558HYPXq1axYsaJS+969ezctWrTgwIEDZBzBeYJ58+axYMEChgwZUpScmzVrRk5OTrETvA0bNmT37t0A5bYTkfD48ENYvBhuugnq1o3cfmMzsQ8d6o1PtW0btvEqM+Oll17ihRdeoEOHDhx//PEkJSXx5z//ucL3jh49muzsbLp168bEiRPp1q0bRx11VMj7/uMf/8ipp55K7969OeGEE0J6z+TJk4umO86ZM4eFCxfSvHlzGjVqxIgRI+jatSv9+/enR48eRe8ZPnw4I0eOJDU1lcTExDLbiUh4TJwITZrAtddGdr/mx0yItLQ0V3LYY82aNXTs2DHisYRDfn4+Bw4cICkpifXr19OrVy++/vpr6kbyT3Q1ieXfi4ifVq+Gzp3h7rvh3nvDs00z+9Q5l1ZRu6ip7hjLcnNzOeecczhw4ADOOaZOnRoXSV1EjtwDD0C9enDDDZHftxJ7GDRs2FC3+hORIps2wZw5MGoUNGsW+f3H5hi7iEgUmzzZm79+883+7F+JXUQkjHbu9OZyDBnizevwgxK7iEgYPfYY7NkD//d//sWgxC4iEia5uTBlClx0EXTt6l8cSuxBEhISSE1NLVo2bNjAsmXLuPHGGwFYtGgRH3zwQVH7l19+uVi53Lvvvpu33347LLEUFg8L9sorrzBhwoSwbF9Ewm/WLNi+HW67zd84NCsmSL169Vi+fHmxdSkpKaSledNGFy1aRIMGDejZsyfgJfaLL76YTp28OwTed9991Rpf37596du3b7XuQ0SOzIED8OCD0LMnnHGGv7Gox16BRYsWcfHFF7NhwwamTZtWdMXnf/7zH1555RX+8Ic/kJqayvr16xk+fHjRpfkpKSmMHz+e7t2707Vr16LKi9nZ2fTu3Zvu3btz3XXX0bZt28N65mV58sknGTNmDOBdRXrjjTfSs2dPjjvuuGIlAR544AF69OhBt27dGD9+fJiPiIiU5vnnvQrifvfWIUp77L/7HZToOFdZair87W/lt8nLyyuqvtiuXTteeumlotdSUlIYOXIkDRo04JZbbgG8HvTFF19c5p2UmjVrxmeffcbjjz/Ogw8+yMyZM7n33ns599xzuf3223nzzTeZPn36EX+mrVu3smTJEr766iv69u3LwIEDWbBgAWvXruWTTz7BOUffvn1ZvHgxZ5111hHvR0TK55xXPqBTJ2983W9Rmdj9UtpQTFUMGDAA8Er0vvjii4BX4rfwD8YFF1xA48al3iI2JP3796dWrVp06tSJbdu2AbBgwQIWLFjASSedBEBOTg5r165VYhepRoWleWfP9m7q5reoTOwV9axjRWGZ3uASveGszRNcBrhwu845br/9dq677rqw7UdEyjdhQuRL85YnCv62xI7gsrel/RyK4BK/CxYs4Pvvvw9rjOeffz6zZs0iJycHgM2bN/Pdd9+FdR8icsgHH8B773lXmdap43c0HiX2Srjkkkt46aWXSE1N5b333mPw4ME88MADnHTSSaxfvz6kbYwfP54FCxbQvXt33njjDVq0aEHDhg1LbdutWzdatWpFq1atuOmmm0Lafp8+fbjiiis47bTT6Nq1KwMHDqz0Hx8RCZ1fpXnLo7K9EbZv3z4SEhKoXbs2H374IaNGjQrruH641ZTfi8iRWLUKunSB8ePhnnuqf38q2xulNm7cyOWXX05BQQF169ZlxowZfockIkeosDRvYBZy1FBij7AOHTrw+eef+x2GiFTRxo2QkQGjR/tTmrc8UTXG7sewkJRNvw+Rsk2e7D2GePoroqImsSclJbFjxw4lkyjhnGPHjh0kJSX5HYpI1Nmxw//SvOUJeSjGzFoDTwHHAAXAdOfcw2bWBHgOSAE2AJc75yo9h69Vq1ZkZWWRnZ1d2bdKNUlKSqJVq1Z+hyESdR57zKvk6Gdp3vKEPCvGzFoALZxzn5lZQ+BToD8wHNjpnJtgZrcBjZ1zt5a3rdJmxYiIxII9e7xe+mmnwauvRnbfoc6KCXkoxjm31Tn3WeD5bmAN0BLoB8wONJuNl+xFROLSrFneUEw0FPsqyxGNsZtZCnAS8DHwU+fcVvCSP3B0uIITEYkmhaV5Tz/dW6JVpac7mlkDYB7wO+fcj2YW6vvSgXSANm3aVHa3IiK+e+45b5rjY4/5HUn5KtVjN7M6eEk9wzn3YmD1tsD4e+E4fKmFSZxz051zac65tObNm1clZhGRiCsszdu5M1x4od/RlC/kxG5e1/wJYI1z7q9BL70CDAs8Hwb8K3zhiYhEh/nzYeVKuPXW6CjNW57KDMWcDvwWWGFmhcVN7gAmAM+b2TXARuCy8IYoIuK/CROgTRsYPNjvSCoWcmJ3zi0ByhpQ7xWecEREos/778OSJfDww9FTmrc8Uf6FQkTEfxMnQtOmcM01fkcSGiV2EZFyrFrlXYh0ww1Qv77f0YRGiV1EpByTJkFycvSV5i2PEruISBk2boSnn4YRI7yhmFihxC4iUoa/BiZ2R2Np3vIosYuIlGLHDpgxA664wpvmGEuU2EVESvHoo9Fdmrc8SuwiIiXs2QOPPAKXXOKVEIg1SuwiIiU88YQ3FHNruXeWiF5K7CIiQQ4cgIcegjPOiO7SvOWpdNleEZF49uyzsVGatzzqsYuIBBQUeOUDunSJ/tK85VGPXUQkYP58r4TAU09Ff2ne8sRw6CIi4RVLpXnLox67iAheWd7334cpU2KjNG951GMXEeFQad6rr/Y7kqpTYheRGm/lSnjtNbjxxtgpzVseJXYRqfEKS/Nef73fkYSHEruI1GiZmfDMM5CeHlulecujxC4iNVqsluYtjxK7iNRY27fDzJkwdCi0bu13NOGjxC4iNVYsl+YtjxK7iNRIhaV5+/aFTp38jia8lNhFpEaaORN27ozd0rzlCTmxm9ksM/vOzFYGrbvHzDab2fLAEsNlc0SkpigszXvmmdCzp9/RhF9leuxPAheUsn6ycy41sMwPT1giItXnmWdg06b47K1DJRK7c24xsLMaYxERqXYFBd4FSbFemrc84RhjH2NmXwaGahqHYXsiItXm9de90ry33gpmfkdTPaqa2KcC7YFUYCvwUFkNzSzdzJaZ2bLs7Owq7lZE5MhMnAht28KgQX5HUn2qlNidc9ucc/nOuQJgBnBKOW2nO+fSnHNpzZs3r8puRUSOSGFp3ptvjv3SvOWpUmI3sxZBP/4aWFlWWxERv02cCM2awTXX+B1J9Qr5Rhtm9gxwNtDMzLKA8cDZZpYKOGADcF01xCgiUmWFpXnvvder5BjPQk7szrkhpax+IoyxiIhUm0mTvFrr8VKatzy68lRE4l5mJjz9NIwYET+lecujxC4ice+vf/WmNsZTad7yKLGLSFzbvh1mzIi/0rzlUWIXkbj26KOQlxd/pXnLo8QuInErJyd+S/OWR4ldROJWYWne227zO5LIUmIXkbi0f7930vSss+C00/yOJrJCnscuIhJLCkvzTpvmdySRpx67iMSdwtK8XbvCr37ldzSRp8QuIvElI4PXjrmW1avh1i1jsacz/I4o4jQUIyLxIyMDNyKdCXlvkcK3DNrxGKTP9F4bOtTf2CJIPXYRiR/jxvFMXj8+pCe38CC1yYfcXBg3zu/IIkqJXUTixvrM2oxkGqezhOv4+6EXNm70LygfKLGLSFzYvx8G151HAvlkMNTrrRdq08a/wHygMXYRiQvjxsGy/Scyr+4Q2u4P6qEnJ8P99/sXmA/UYxeRmPfmm/DggzBqFAyYdbF3U1Mz73H69Bp14hTAnHMR32laWppbtmxZxPcrIvFn61Y48UQ45hj4+GOoV8/viKqPmX3qnEurqJ167CISswoK4MorvWJfzz4b30m9MjTGLiIxa9IkePttb7SlJlVvrIh67CISkz76CO68Ey67DK691u9ooosSu4jEnF27YMgQ745I06d750nlEA3FiEhMcQ7S073KjUuWQKNGfkcUfZTYRSSmzJwJL7wAf/kL/OIXfkcTnTQUIyIxY/VqGDsWzjuvZt3DtLJCTuxmNsvMvjOzlUHrmpjZW2a2NvDYuHrCFJGaLi8PBg2CBg3gqaeglrqlZarMoXkSuKDEutuAd5xzHYB3Aj+LiITdzTfDypVeUm/Rwu9oolvIid05txjYWWJ1P2B24PlsoH+Y4hIRKTJvHkydCrfcAheU7F7KYar6ZeanzrmtAIHHo6sekojIIZmZ3jz1Hj1qXC2vIxaxUSozSzezZWa2LDs7O1K7FZEYdvCgV78rP9+7OXXdun5HFBuqmti3mVkLgMDjd2U1dM5Nd86lOefSmjdvXsXdikhNcO+98P77MG0atG/vdzSxo6qJ/RVgWOD5MOBfVdyeiAgA777rDb1cdRVccYXf0cSWykx3fAb4EPi5mWWZ2TXABKC3ma0Fegd+FhGpkuxsbwjm+OPhkUf8jib2hHzlqXNuSBkv9QpTLCIiOOf10nfsgPnzoX59vyOKPSopICJR5eGH4fXXYcoUSE31O5rYpGu3RCRqfPaZVyqgb18YM8bvaGKXEruIRIXdu2HwYDj6aJg1S6V4q0JDMSISFa6/Htav92bDNG3qdzSxTT12EfHdP//pLXfdBWed5Xc0sU+JXUR8tXYtjBoFZ57p3epOqk6JXUR8s2+fN65ety5kZEBtDQ6HhQ6jiPjmttu8mTAvv+zdv1TCQz12EfHF66/D3/7mTWvs18/vaOKLEruIRNyWLTB8OJx4IjzwgN/RxB8ldhGJqPx8+M1vIDcXnn0WkpL8jij+KLGLSGRkZEBKChNq38m778IjQz/khBP8Dio+KbGLSPXLyID0dN7PbMl47mEIT3PVnPO89RJ25pyL+E7T0tLcsmXLIr5fEfFJSgrfZebSg6XU5iCfcxI/YTe0bQsbNvgdXcwws0+dc2kVtdN0RxGpdjsycziPhWynGYs420vqABs3+htYnNJQjIhUqx9+gPPrLuRrjucV+tKDoG/rbdr4F1gcU2IXkWqTkwMXXghf5nfmxcQr6MXCQy8mJ3v3vpOwU2IXkWqRl+fVVf/4Y3jmuQQufOJSb0zdzHucPt27/52EnRK7iIRPYErjPktiQJNFLFrkeOopuPRSvCS+YQMUFHiPSurVRoldRMIjMKXxQOZmBvMMb+49m5l1RnOF05TGSFNiF5Gqy8iAYcPYl3uQy3iBl/k1jzCGq/dPg3Hj/I6uxtF0RxGpmkBPPTe/LgN4kX9zAY8whjE85r2uKY0Rp8QuIlUzbhw7cxO5hFf5kNOYyTVcw6xDr2tKY8QpsYtIlWzKLOAC3mMdP+M5BnEZcw+9qCmNvgjLGLuZbTCzFWa23MxUK0Ak3gVmv6yyLpzGB2TRin9zfvGknpCgKY0+CWeP/Rzn3PYwbk9EotHo0TBtGktcTy7hVeqRx2LO4kS+PNQmOVlJ3UeaFSMioRs9GqZOZa4bQG/e4mi+4wN6ekk9IUEXH0WJcPXYHbDAzBzwd+fc9JINzCwdSAdoo5MpIrEnI4OCqX/nz4zjLv5ET97nX/SjGTu81wsKvEV8F67EfrpzbouZHQ28ZWZfOecWBzcIJPvp4JXtDdN+RSRCfrh9AsOYx7/oz2/4JzMYQRL7DjVQhy1qhGUoxjm3JfD4HfAScEo4tisi0WHlSuixaR6vcxF/YyxPcWXxpG6m2S9RpMqJ3czqm1nDwudAH2BlVbcrIj467zwvWZvxnA3i1BPz2J3QiIWcy1imYCXbjxypMfUoEo4e+0+BJWb2BfAJ8Lpz7s0wbFdE/NC5M7zzDgeozU08xGCe46SCT/nsuMs4M/mz4m3NYNQoePxxf2KVUlV5jN059w1wYhhiERG/ZWTA6tV8SwpDyeBDenIDU3iQW6i79gDMmePVftm40RtTv/9+9dSjkO55KiJFXNsUnth4Hr9nMrUoYDrpDOL5oAaa9+CnUO95qnnsIjVZ4ApSatViS6tTuGjj44xgJj1Yypd0K57UJWaoVoxITRWoyuhyc3mGIYzZ/Ch7SeIRxjCax6lFid55r17+xCmVph67SE2SkQHNmnknPX/zG7JyGzOAFxnK05zAV3zBiYyxUpJ6p07w9tv+xCyVpsQuUlNkZMCVV8KOHRwkgYe5kY6s4d+czyT+wHucSQfWeePowfcmnTMHVq3yO3qpBA3FiNQUI0dCQQH/4Sxu4BFW0I0LeIPHuJ7j+PZQu7ZtvXuSSsxSj10kngWdHP0qpyX9eJmz+Q8/cBRzuZT5XFg8qat+elxQYheJV4GTo9sy87jePUIXVvIu5/BnbmcNHbmUF4tfQar66XFDQzEicWrzrVN4IPd+ppPOfuoykmnczX0cTfbhjevWhVmzlNTjhHrsIvEgaMgls9XpjDrva47bvJhHGcMgnmMNHXmUG0pP6klJSupxRj12kVgXGHL5Krc1DzCDpzZfiW12XJX4DLftu4d2bCj9fQkJkJ6uOi9xSIldJNZkZBTVaznYuh2v7jydx3L/xTucRxJ5jGIq/8ckWjXYCwl5kBv0Xt2yrkbQUIxILAn0zjdmFvAndwftNi5iQM5TfM3x3M8dZNKWKYylFZth504viQfPSVdSrxHUYxeJEd9+C3PHZDI3dyGfcCoAvVnAI9zAxbxGbfKLv6FNGy+JK5HXOOqxi0SLoBOgpKRARgZr18KECZCWBscdB/+36w4KqMUEbmUd7VnA+fTnX4cndc1Hr9HUYxfxS+FYeWaml8wDN4L+ip8zN3Mgc3/blS8CJVtOPRUeeAAunXwG7ba8f/i2mjaFBg1UJ10A9dilpOAiUWbe84yMQ6+V6FFKCEaPhtq1veOZkOAlYDP47W8hM5NtHM3cgl8zhkc4gTV05Cvu4k80cD8yufF9ZGbCRx/BLbdAu0mjvN54sORkePhhrwxAQYH3qKResznnIr6cfPLJTnwwZ45zbds6Z+Y9zplz+Ot16zrnlYE6tNSp49yoUc4lJxdfn5x8+DYq2m/Tpt5S8nlwPBXFGW3Ki7dXr2LHLI9Et5ST3eOMdMP4h/s5aw4dTnLcr3jdTWGMy+JYb6VZ5fYncQ1Y5kLIsbqDUiwKmu5Gkybeuh07vN5gfr43+6HkV/HAbApyvblvBRh76jVn96Sp7O49gN27YffFQ9i9bQ+7acge6nOAOhRQiwJq4SyBAgcOoxYFJLKPJPaS2LQhSX9/mMRESEz0rnUJfkx842WS7riJxLzvSSaXRPaX/bmSk2HYMJg9uyhOwLsqsmFDb5ZHkyawdy/s2eO91rSp11str4caPORReIzq14e8vKLhj1LVqgXnnAPr1hV/b+HxBRg71jv2JT5HzpRZrN/WgPXjnmAtHVhJF5aTyho6kh8YAT2abfyCj+jJB/yS/3Ayn1KHg8W3pYJcEiTUOygpsceaEgm6pIMksJUWbErsQNaIe9mUciZZWbBp+nyycpuwmZbsohE5NIxw4J467KcBOTRkNz/hx6LHosVy+InbVfRzcLvg5Sf8SF32e7VOyrscvoLjVVkO2E1Dsuu0JLugKdn5jdnCsWymJZm0ZT3tWcfP2MYxxd7XkixO5AtSWc5JfE53PqMd3xav1VKS5pxLCUrs8SK4d96mDeTkwI4d5FKPL+nGSrqwki6sojNfczxZtKKAhGKbqF8fWu9ZQyuyaMlmmrAzKEnm0DBjGg0bQsMRg2mwbR0N2U0DcqjDARLIx3DUqmVYwUEMRwG12Ecie0li37HHse/Nd9m7F/bto+ix6PnQq9kbaJtHPXJoUCxFF6b04s9/EtKhqUU+9cjzloT9JB3Xknr1KFqSkqDef96kXt4O6pFHIvtIIL/oMxku8F8jnwQOUpsD1GEfieRRjz3UL1pyaMAuGvE9jdlP4mGxGAW0YCs/Y11gWc/PWEd71tGe9RzFj5X7vYfyTURqnFATu2bFRIOMjOJf6Qv/pwZIT+dA7n5WciKfZJ7CUnqwlB6sonPRV/pk9tCJ1ZzJe7TjW1qzidZsohWbaf39lxx1FFi7X3nDCSW1bQtXBJ4/dAlcfTXsLzFcUqcOXHvt4UMkyckw6c/QtZzPdsfC0vdbloQECvILyKEBP3DUYck/eMklmTzqeX808pPJO/kq8vK8EZa9e73DmZfXkjx+Rh712EdiIK0nFCX0wgRfmPDrsr9omKkwrR/NdxzHNzRiF435nuZk05xsmrGd5mRzLFs4hv8Vn3LYtq33WNFnN/OG2MsbRhOprFAG4sO96ORpkFJOWOZj7ova3d1DyXe6X/G6Syan6OUmbHfn84a7i3vdy/R135Di8rHDT3iCd2IteD+hnPycM8c7oVnYpmnTqp3ULG2/ZS3JyaWfpA1lCf6swdq2rfy2qroUHteyPntiok58yhEhxJOnSuyRVjI5BpLodzRzsxjuhpDhjuZ/RTng56xxo3nUPcMgt552rqCyyaW8fUcqqVRlVkzTpt6snPI+a926ZX+WyvxhqcxSt27pcQX/ISz5WZTIpYoimtiBC4D/AuuA2ypqX2MTe4kks4Vj3GOMcufwjqvFQQfO/ZStbij/dP9gmNtIq9KTStOmhydKcC4hwRX1XuMpgZRM9PXrl51Iy3t/8DGqX9+5WrXKT961annTFUu+t/D4KmlLhIWa2Kt88tTMEoCvgd5AFrAUGOKcW13We2rsydOUFPZkZvMcg/gHV/E+p+OoxQmsYSBzGcCLpLL80EyJpk29AeOS49qaKSFSI4V68jQcV56eAqxzzn3jnNsPPAv0C8N2Y1MZV2euWAFjMv/AsWzhGmaxg6bcwz2sohNr6MQfuZuTgpN6nTreCVRV5xORSgrHrJiWwKagn7MgUHouiJmlA+kAbdq0CcNuo1CJOdN5mdt44epFTLu3Dx+ubU4i1zKQFxjJNE7n/eI9czh8VkxhAlciF5FKCEdiL+0ai8PGd5xz04Hp4A3FhGG/0WfcOMjNJZM2/I3fMZszZyZ3AAAJi0lEQVRhfL+/Ccd/u56HHmrOsORXaHrzdYcPrWi+soiEUTgSexbQOujnVsCWMGw35vwvcx/3M4XppOMwLmUe1/F3fnlwMXZTAXAZNNxf/IIjzVkWkTALR2JfCnQws3bAZmAwhy55qRF27oRJk2CKfcN+V4ermcVd/JHWZHkNCi9WAd34QESqXZVPnjrnDgJjgH8Da4DnnXOrqrrdqFPKSdHdu+G++6BdOy+xDzjtf3yVdBLTue5QUtcND0QkwsJSj905N985d7xzrr1zLv6yWOFJ0cxMcI68zG08dNVK2h27l/Hj4dxz4csvYc777fjZzNs0i0VEfKUiYKFISYHMTA5Qm5lcy5+4ky20pE/SYv60+Cx69PA7QBGpCSI5jz3+bdzIMk6mB0sZzVTa8S2L+CX/3ne2krqIRB0l9grs2QO3NJzGqXzMdxzNPAbwHmfySxZ7s1pERKKMEnuwEidI3779Hbp2hYd+TGdE7X+wmk4M4CVv4r5OiopIlFJiLxR0gnSna8RVmePpPaEXtfN+ZNEimPZkPRq1baSToiIS9XTytFDgBOm7nM1gnmUHTfk/JnFX69nU2/hfv6MTEdEdlCpt40bmcilDyeBnrOPfnE8qX0BWuXelFBGJOhqKCZja+A4u53l6sJQlnOElddAJUhGJOTU+sTsH99wDo3f+iYtqvckC+tCYXd6LOkEqIjGoRif2/HwYPRruvReuugpe+scuktserROkIhLTauwY+9698JvfwLx5cOut8Je/gNkVcGWNql8mInGoRib2H3+Efv1g0SL461/h97/3OyIRkfCpcYn9f/+DX/0KVq6EOXM00iIi8adGJfb166FPHy+5v/oqXHCB3xGJiIRfjUnsn3/u9dQPHoSFC+HUw+7KKiISH2rErJh334Vf/hISE2HJEiV1EYlvcZ/Y5871hlzatIH334cTTvA7IhGR6hXXiX3aNLj8ckhLg8WLoVUrvyMSEal+cZnYnfMuOho1Ci66CN56C5o08TsqEZHIiLuTp/n5cMMNMHUqDB8OM2ZA7bj7lCIiZYurHvu+fTB4sJfUb70VZs1SUheRmidu0t6PP0L//t4MGF1NKiI1WVwk9m3bvDnqK1bAP//p1YAREampqjQUY2b3mNlmM1seWC4MV2Ch+uYbOP10+O9/vatJldRFpKYLR499snPuwTBsp9KWL/fmqOtqUhGRQ2L25OmiRXDWWVC3rq4mFREJFo7EPsbMvjSzWWbWOAzbK11GhnfD6Vq1mNd8JOf3zqd1a/jgA11NKiISrMLEbmZvm9nKUpZ+wFSgPZAKbAUeKmc76Wa2zMyWZWdnVy7KjAxIT4fMTP7uRnDZ9sdJK/iE9258QVeTioiUYM658GzILAV4zTnXpaK2aWlpbtmyZaFvPCUFMjP5C7dxB3/hIl7jeS73bmO3YcMRRiwiElvM7FPnXFpF7ao6K6ZF0I+/BlZWZXtl2rgRgA6s5Wqe4CV+TTJ5RetFROSQqs6KmWRmqYADNgDXVTmi0rRpA5mZDGQeA5lXfL2IiBRTpcTunPttuAIp1/33e2PsubmH1iUne+tFRKSY2JjuOHQoTJ8ObduCmfc4fbpuWCoiUorYKSkwdKgSuYhICGKjxy4iIiFTYhcRiTNK7CIicUaJXUQkziixi4jEmbCVFKjUTs2ygcygVc2A7REPpPIUZ3jFQpyxECMoznCK5hjbOueaV9TIl8R+WBBmy0Kpf+A3xRlesRBnLMQIijOcYiHGimgoRkQkziixi4jEmWhJ7NP9DiBEijO8YiHOWIgRFGc4xUKM5YqKMXYREQmfaOmxi4hImPiS2M3sATP7KnCv1JfMrFEZ7S4ws/+a2Tozu82HOC8zs1VmVmBmZZ4lN7MNZrbCzJabWSVuDRUelYjTt+NpZk3M7C0zWxt4LPX+uGaWHziOy83slQjGV+6xMbNEM3su8PrHgTuGRVwIcQ43s+ygY3itDzHOMrPvzKzUG++YZ0rgM3xpZt2jMMazzeyHoON4d6RjrBLnXMQXoA9QO/B8IjCxlDYJwHrgOKAu8AXQKcJxdgR+DiwC0spptwFo5sexDDVOv48nMAm4LfD8ttJ+54HXcnw4fhUeG2A0MC3wfDDwXJTGORx4NNKxlYjhLKA7sLKM1y8E3gAM+AXwcRTGeDberT59O45VWXzpsTvnFjjnDgZ+/Ago7ZbUpwDrnHPfOOf2A88C/SIVI4Bzbo1z7r+R3OeRCDFOv49nP2B24PlsoH8E912RUI5NcPxzgV5mZhGMEfz/HYbEObcY2FlOk37AU87zEdCoxG02q10IMca0aBhjvxrvr3dJLYFNQT9nBdZFIwcsMLNPzSzd72DK4Pfx/KlzbitA4PHoMtolmdkyM/vIzCKV/EM5NkVtAp2SH4CmEYmulBgCyvodXhoY4phrZq0jE1ql+P1vMVSnmdkXZvaGmXX2O5jKqLYbbZjZ28Axpbw0zjn3r0CbccBBIKO0TZSyLuxTeEKJMwSnO+e2mNnRwFtm9lWgRxA2YYiz2o9neTFWYjNtAsfyOGChma1wzq0PT4RlCuXYROTfYwVCieFV4Bnn3D4zG4n3LePcao+scqLhWFbkM7zL93PM7ELgZaCDzzGFrNoSu3PuvPJeN7NhwMVALxcY1CohCwjubbQCtoQvQk9FcYa4jS2Bx+/M7CW8r8xhTexhiLPaj2d5MZrZNjNr4ZzbGvja/V0Z2yg8lt+Y2SLgJLxx5eoUyrEpbJNlZrWBo4j8V/kK43TO7Qj6cQbeOaxoE5H/t6vCOfdj0PP5Zva4mTVzzkVrDZli/JoVcwFwK9DXOZdbRrOlQAcza2dmdfFOWEVslkSozKy+mTUsfI53YrjUM+0+8/t4vgIMCzwfBhz2LcPMGptZYuB5M+B0YHUEYgvl2ATHPxBYWEaHpDpVGGeJseq+wJoIxheqV4ArA7NjfgH8UDhMFy3M7JjCcyhmdgpertxR/ruiiB9nbIF1eGNsywNL4WyDY4H5Qe0uBL7G67GN8yHOX+P1LvYB24B/l4wTb4bCF4FlVbTG6ffxxBuPfgdYG3hsElifBswMPO8JrAgcyxXANRGM77BjA9yH1/kASAJeCPzb/QQ4LtK/5xDj/Evg3+EXwLvACT7E+AywFTgQ+Hd5DTASGBl43YDHAp9hBeXMOPMxxjFBx/EjoKcfv+8jXXTlqYhInImGWTEiIhJGSuwiInFGiV1EJM4osYuIxBkldhGROKPELiISZ5TYRUTijBK7iEic+X8zvwTIsWCt9QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x28289013828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 4.评估模型性能\n",
    "# %pylab    \n",
    "model.eval()                     # 模型变成测试模式\n",
    "model.cpu()\n",
    "test_x, test_y = get_batch(64)\n",
    "test_x, test_y = test_x.cpu(), test_y.cpu()  # 将数据放到cpu上\n",
    "predict = model(test_x)          # 对输入数据的输出进行预测\n",
    "predict = predict.data.numpy()   # 转换为numpy数据才可以画图\n",
    "test_x = test_x[:,0].unsqueeze(1)\n",
    "plt.plot(test_x.numpy(), test_y.numpy(), 'ro', label='Original Data')\n",
    "# 由于数据大小顺序是乱的，故无法画出曲线？？？\n",
    "# plt.plot(test_x.numpy(), predict, 'b.',label='Fitting Data')\n",
    "plt_data = np.concatenate((test_x.numpy(), predict), axis=1)\n",
    "# 将数据按第一行排序后绘制直线\n",
    "plt_data = plt_data[plt_data[:,0].argsort()]\n",
    "plt.plot(plt_data[:,0], plt_data[:,1], 'b-', label='Fitting Line')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 分类问题"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Logistic回归\n",
    "Logistic回归中，输出y=1的对数概率是x的线性函数：\n",
    "\n",
    "1. 拟合决策边界(线性或多项式都行)\n",
    "2. 建立决策边界和分类概率的关系，得到分类概率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from torch import nn, optim\n",
    "# %pylab inline\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3X+UXHWZ5/H3kx9MaEQhncAGmnTCIQeyBJKB5tewRofIoDMIjqIHzEBQzuTsWXbFYVeHOawronHBdY/DHDyyQVQkPfxcZ4juGM1JdGbHs4IdTSAJcMKvJA2RNB1AEIGAz/5xb0N1UVVdXXV/fO+9n9c5darr9u2qp6tuPfd7n+/3fq+5OyIiUl5T8g5ARETSpUQvIlJySvQiIiWnRC8iUnJK9CIiJadELyJSckr0UjlmdpOZfS6h57rGzNYk8VwiaVGil0IxsyfN7H3dPIe7/3t3/2JSMbXLzL5jZl/K+nVFlOilVMxsWt4xiIRGiV4Kw8xuA+YC3zezl8zss2Y2z8zczC4zs13Axnjdu83s12b2gpn9i5kdX/M8b7aszey9ZjZsZv/ZzPaa2R4z+0SLGOab2T+b2Ytmth6YVff7hq9rZiuB5cBn49i/Hy+/yswei59vu5n9ebLvmogSvRSIu18M7AI+6O7vcPev1Pz6PcBC4Jz48Q+BBcBhwC+BwRZP/W+AdwFHApcBXzezQ5us+/fAJqIE/0VgRd3vG76uu6+Of/5KHPsH4/UfA94dv/4XgDVmNqdFrCKTpkQvZXGNu//W3X8H4O7fcvcX3f1V4BpgsZm9q8nf7geudff97v5PwEvAsfUrmdlc4BTgc+7+qrv/C/D92nUm+bq4+93u/rS7/97d7wR2AKdO8n8XaUmJXspi99gPZjbVzK6LSyK/AZ6MfzWr4V/CqLu/XvP4ZeAdDdY7AnjO3X9bs2xnF6+LmV1iZpvN7Hkzex5Y1Gp9kU4o0UvRNJtutXb5x4HzgfcRlUTmxcuty9feAxxqZgfVLJs7idcdF7uZ9QM3A/8R6HX3Q4CtCcQpMo4SvRTNM8DRE6xzMPAqMAr0AF9O4oXdfScwBHzBzA4ws38HfLBmlYletz72g4iS/whA3Am8KIlYRWop0UvR/Hfgv8aljv/SZJ3vEpVUngK2Az9P8PU/DpwG7AM+H79Wu697C/Bv49j/0d23A/8T+H9EO4ETgJ8lGKsIAKYLj4iIlJta9CIiJadELyJSckr0IiIlp0QvIlJyQUwANWvWLJ83b17eYYiIFMqmTZuedffZE60XRKKfN28eQ0NDeYchIlIoZrZz4rVUuhERKT0lehGRkpsw0ZvZt+J5urfWLJtpZuvNbEd8f2i83Mzs78zsUTN7wMxOSjN4ERGZWDst+u8A769bdhWwwd0XABvixwAfIJqLewGwEvhGMmGKiEinJkz08Zzb++oWnw/cGv98K/ChmuXf9cjPgUN0EQURkXx1WqM/3N33AMT3h8XLj6RmXnBgOF72Nma20syGzGxoZGSkwzC6NDgI8+bBlCnR/WCrixCJiBRT0p2xjebRbjhrmruvdvcBdx+YPXvCYaDJGxyElSth505wj+5XrlSyF5HS6TTRPzNWkonv98bLh4GjatbrA57uPLwUXX01vPzy+GUvvxwtFxEpkU4T/VreuijyCuDemuWXxKNvTgdeGCvxBGfXrsktFxEpqHaGV95OdGGEY81s2MwuA64DzjazHcDZ8WOAfwIeBx4lukTaf0gl6iTMnTu55QlRt0A16XOXPE04BYK7X9TkV8sarOvA5d0GlYlVq6KafG35pqcnWp6SsW6BsZcc6xYAWL48tZeVnOlzl7xV98zY5cth9Wro7wez6H716lS/eeoWyE+eLWp97pK36iZ6iJL6k0/C738f3afcvFK3QHZqE/usWfDJT+Y3wKoIn7tKS+VW7USfsZy6BSqnfuTs6Ci89tr4dbJsUXf6uWeVfDXSuPyU6DO0alXUDVAr5W6BSmpUKmkkqxZ1J597lslXpaXyq0aiD+S4NIdugUpqN4FndSTVyeeeZfItQmlJumPRQJl8DQwMeGoXHqkf8gBRc0oZtrTmzYtawK2EvglMmRK15OuZRV1KSWr2fvX3R11XEi4z2+TuAxOtV/4WvY5LK6dRqWT6dOjtLc6RVJb9OSopll/5E72OSyunUank29+GZ5/NbIBV17JMviopll/5Szc6LpWCGhyMDjx37Ypa8qtWKfnKeCrdjNFxqRRUxqd5SImVP9HruFREKq78iR7UNJIgBTLqVypgwknNRCR5muhMslSNFr1IYDTqV7KkRC+SgyKN+lWJqfiU6EVyUJQJ7jThWTko0YvkoCijflViKgclepEcFGXUb7NS0kRzCUlYlOhFclKEUb/NSklmKt8UiRK9SIkk3XG6alWU1Ou5q3xTJF0lejO7wsy2mtk2M/t0vGymma03sx3x/aHJhCoiraTRcbp8eePpkiHMEULSWMeJ3swWAX8JnAosBs41swXAVcAGd18AbIgfi0hKxlrxf/EX6XSc9vc3Xh7aCCFprpsW/ULg5+7+sru/Dvwz8OfA+cCt8Tq3Ah/qLkQpKo2/Tl9tK76ZblveRRkhJM11k+i3AkvNrNfMeoA/BY4CDnf3PQDx/WGN/tjMVprZkJkNjYyMdBGGhEjjr7PRzvVxu215F2WEkDTXcaJ394eA64H1wDpgC/D6JP5+tbsPuPvA7NmzOw1D2pBHy1rjr7MxUWs9qZZ3EUYITaTKR5hddca6+y3ufpK7LwX2ATuAZ8xsDkB8v7f7MKVTebWsi3SKf5G1aq2r5f2Wqh9hdjvq5rD4fi7wYeB2YC2wIl5lBXBvN68h3cmrZV2UU/yLrln9fM2a4ra801D1I8xux9H/bzPbDnwfuNzdnwOuA842sx3A2fFjyUleLWt14GWjVf28yqWK+v+9WWd1ZY4w3T3328knn+wSWbPGvb/f3Sy6X7Omu+fr73ePDlbH3/r7u491Ikn/L9K+NWvce3rGf+Y9PdX4DBr972b5fQ/SBAx5Gzk29yTvSvRvSuPLWeUvfJXluYPPW7P/vT7Zp/k9yKqRo0RfQGl9OdWyrp5mLVizvCNLX7P/fey7lPb3IMvGVbuJ3qJ18zUwMOBDQ0N5h5G7KVMan25uFg1rk+YGB6OOtV27og7fVauq3RHZrC7d3x910pZZ3v97lq9vZpvcfWCi9TSpWUA0UqUzVR86V2usE3LnzrdPRlaVzvC8BwKEOLRYiT4geW+gRRXi0Lk8RrzUT4fg/layr9KY+rzP5A2ywdZOfSftm2r0b1E9ffJCq0fn1QFe5Q7YkKhG34Rq9NKNvGuy9bKMp7ZvotlXWX082cuqz0g1eqmM0EpeWdVo6/smmlEfT/ZCmxtIiV4KL++abL2sarTtzFypPh4BJXopiZBaUFkdYbQ6QghhhyfhmJZ3ACJlM5ZY067Rzp0bVt+EhEstepEUZHGEEVrfhIRLiV6koELrm5BwqXQjUmDLlyuxy8TUohcRKTklehGRklOiFxEpOSV6kViVL70n5abOWBHemk5g7EzTsamOQZ2dUnxq0YsQ5lTHIknpKtGb2V+Z2TYz22pmt5vZDDObb2b3mdkOM7vTzA5IKliRtIR4sQiRpHSc6M3sSOBTwIC7LwKmAhcC1wNfc/cFwHPAZUkEKpKmIC8WIZKQbks304ADzWwa0APsAc4C7ol/fyvwoS5fQyR1mk5AyqzjRO/uTwFfBXYRJfgXgE3A8+7+erzaMHBko783s5VmNmRmQyMjI52GUS4a9pEbTScgZdZN6eZQ4HxgPnAEcBDwgQarNrwkgruvdvcBdx+YPXt2p2GUh65wnbsQpjrWvl7S0E3p5n3AE+4+4u77ge8BfwQcEpdyAPqAp7uMsRo07KPytK+XtHST6HcBp5tZj5kZsAzYDvwEuCBeZwVwb3chZiTvppSGfeQu701A+3pJSzc1+vuIOl1/CTwYP9dq4K+BK83sUaAXuCWBONMVQlNKwz5yFcImoH29pKWrUTfu/nl3P87dF7n7xe7+qrs/7u6nuvsx7v5Rd381qWBTE0JTSsM+chXCJqB9fTjyPrpLms6MhTCaUhr2kasQNgHt68MQwtFd0pToIZymVAjDPioqhE2gyPv6MrWAQzi6S5oSPaTblCrTN6DEQmlNF3FfX7YWcAhHd4lz99xvJ598suduzRr3/n53s+h+zZpknrOnxz3a/qNbT08yz52BNN6SbqUZU4j/bxH094/fxMdu/f15R9aZIv0/wJC3kWNzT/IeSqJPQ5G2mDoh7qNCjKlsOtnZmTXezM3SjjYdRdrOlOjblWYzrsDfgBD3USHGVCadJrgyfi5FObprN9FbtG6+BgYGfGhoKPsXrr/aBESF2aR6wObNiwqW9fr7owJswKZMib6u9cyi+nEeQoypTDrdXNP+GklzZrbJ3QcmWq/anbFpd6+H0sPXgRBGobT72hpnnoxOOyGLPFqoKqqd6NPuXi/wNyDEfVSIMZVJsx3mlCkTDxor4mihSmmnvpP2LbcafRmLiwkKsU4ZYkxl0ahGX38LtVOyqlCNvg0qLoqMMzgYVS537Ypa8W+88fZ1CtDFVBmq0bejndKKTngqDH1U3astwTTr4C70iUMVVe0W/UTU4i+MRh/VAQfAwQfDvn1R/XnVKn1sk1HgQWOVoRZ9Eso46UVJNfqoXnsNRkfLcVp+HtT5na4sj0CV6Fsp5aQX5dTOR6J99OQUeNBY8LKeH0ilm1Z07FoYzT6qejq5SkKQVGpR6SYJSR+7qrcwNY0+qkZ0cpWEIOtigRJ9K0keu5ZtLtfA1H9Uvb0wffr4dVRfllBkfZa3SjdZURkoc7VjwjXqRkKS1IA+lW5Co47dzOm0/LdT9TAMWXd0d5zozexYM9tcc/uNmX3azGaa2Xoz2xHfH5pkwIWlGblyV7YkN9n/p2zVw6J/npk2RNqZJ2GiGzAV+DXQD3wFuCpefhVw/UR/X9oLj9Qq0tUM2lC0OWdK9vZ39P+UaWqnsn2enSLLC48AfwL8LP75EWBO/PMc4JGJ/r4Sid69eNmxiSJ+ycqU5Nw7+38KfB2ctynb59mpdhN9UjX6C4Hb458Pd/c98dHCHuCwRn9gZivNbMjMhkZGRhIKI3AlKRoX8YThsnWRdPL/NKsSuhev9BHy59mqpJRbuamdvUGrG3AA8CxRggd4vu73z030HJVp0ZdEEVuGZWsBdvL/TDQNcehHZbVC/TxbHe2mcSRMVqUb4HzgxzWPVbopuVC/ZK0UsdzUSqf/z1j1sFmyD/kzrBXq59nqu5HG9ybLRH8H8Imax/+D8Z2xX5noOZToiyXUL9lEStJF8qZu/p8iHpXVC/HzbPW+pvGet5vouzphysx6gN3A0e7+QrysF7gLmAvsAj7q7vtaPU8lTpgqGZ2MVGw6fy8drd5XSP49z+SEKXd/2d17x5J8vGzU3Ze5+4L4vmWSl2IqSb9yZWkK4nS0el/zfM91ZqxIBWkK4nS0el/zfM81101RqXYSDH0Ukpd2SzfTsghGElY/I9LYueygDJMxfRRSBCrdFFERz1gqKX0UYSj6vDdpU6IvopBPC6wYfRT5S2uytjLtPJTok5bF1qGZMIOhjyJ/aRxVlW2mTyX6JGW1dRR8bFyZWkoF/yhKIY2jqtKV5No5qyrtW2nOjM1yboAQTwtsQ1HPqm2loB9FaaTxtSvKmcNkcWZsUkozvHLKlGh7qGcWnVkkOiNTEpfUZflqFWU71aUE86CC7YTUeSlJS+NEpLKV5JTok1S2rSMF2hdKGpKekqNsZw4r0SepbFtHKx32qGpfKEVRpvmcdGZs0sYmtSizLk4HHfu1pgwQyY46Y2XyitJTJVJy6oyV9KhHVaRQlOhl8tSjKlIoSvQyeepRFSkUJXqZvCqNLmqiTNM4SHby2m6U6KUzY2PPbrstenzxxZXJeGWb8Eqyked2o0Q/Rk20yatoxivdhFeSiTy3m64SvZkdYmb3mNnDZvaQmZ1hZjPNbL2Z7YjvD00q2NRUNGG9zWR3dhXNeBp0JJ3Ic7vptkV/A7DO3Y8DFgMPAVcBG9x9AbAhfhy2iiascTrZ2aW05YZ+cKVBR9KJXLebdqa4bHQD3gk8QXzSVc3yR4A58c9zgEcmeq7cpykuypykaepkrtcU5octwjTGRYhRwpPGdkOb0xR3k+iXAPcD3wF+BXwTOAh4vm6955r8/UpgCBiaO3du5/9pErKcRz5UnezsUthyi/JRaA566UTS2027ib7jKRDMbAD4OXCmu99nZjcAvwH+k7sfUrPec+7esk6f+xQIaUxoXTSdTmswOJjoxDWa0l+kfVlMgTAMDLv7ffHje4CTgGfMbE4cxBxgbxevkQ2NC+/8JKiEp/hT/VskeR0nenf/NbDbzI6NFy0DtgNrgRXxshXAvV1FmJUyzUnaiUB2djrpViR5Xc1eaWZLiGrzBwCPA58g2nncBcwFdgEfdfd9rZ4n99KNBCXhapBIabVbutE0xSKSG+3Uu9NuoteFR0QkF11cv0YmSVMgFF3oZxdJ5TXbRHWeYnbUoi8yNYkkcK02UU0lkR3V6ItMl/STwLXaREGbb7d0KcEqUJNIAtdqE9VQ2uwo0ReZzi6SwLXaRAM5daMSlOiLTE0iCdxEm2jVz1PMihJ9kalJJIHTJhoGdcaKiBSUOmNFRARQohcRKT0lehGRklOiF0mRZqiQECjRSzYqmPE6ud66SBqU6CX9JBxKxst4Z6NJuyQUGl5ZdVlcLzeEOXlyuC6wrn8radOFR6Q9WSThEDJeDjubEPZvUm4aRy/tyWJitBDm5MlhAjjNUCGhUKKvuiyScAgZL4edjU7/l1Ao0VddFkk474w3OAgvvfT25RnsbDRpl4Sgq0RvZk+a2YNmttnMhuJlM81svZntiO8PTSZUSUWrJJzkKJW8Mt5YJ+zo6Pjlvb1qXktlJNGi/2N3X1LTIXAVsMHdFwAb4sfJq+C47NQ0SsKhDInsVqMxjgDveIeSvFRGV6NuzOxJYMDdn61Z9gjwXnffY2ZzgJ+6+7GtnmfSo25yGCpXOWUZMhLCiB+RlGQ16saBH5vZJjOLL/nL4e6+ByC+P6xJgCvNbMjMhkZGRib3qjoTJX3NRqM0Sv4hC2HEj0jOuk30Z7r7ScAHgMvNbGm7f+juq919wN0HZs+ePblX1bVS09csEZoVq3wTwoifilJ1NRxdJXp3fzq+3wv8A3Aq8ExcsiG+39ttkG+jVlr6Vq2Kkno992IdOeU94qeiytLFUxYd1+jN7CBgiru/GP+8HrgWWAaMuvt1ZnYVMNPdP9vquVSjD1SjRD+2XPVtaaEsXTyhy6JGfzjwr2a2Bbgf+D/uvg64DjjbzHYAZ8ePk6VWWjb6+xsv15FTIWVZSlF1NTDunvvt5JNP9qCsWePe3+9uFt2vWZN3RPlYs8a9p8c9OvqObj091X0/Cizrj7K/f/xrjd36+9N5vaoChryNHKszY+upuPiWRkdOK1ZENXr1sDUWaA9k1gPV1AcemHb2BmnfgmrRqynSnFr4reX4/kx0EGrWeLM2yy8m6R5ttug1TXE9nWDTnHrYWsvp/WlnbII+unLSNMWd0tDNxgYHm58spR62SE49kO2UZVRK6Vyg1bhJUaKvp2/E2401GZup+k5wTE6NhHb2L1UYqJZGQi5Nl1079Z20b0HV6N1VXKzXrN9CNfrxcqrRq1spvbc+9PeWNmv0uSd5DzHRy3jNevJASb5eDo2EtPcvRWj3pJWQ8+jEnox2E706Y2Vi6skL3uBgVJPftSuqFK1alUxZpignoac1hiL0Tb8anbFJFOXK0NOSNvVbBC+t67oUZaLYtLpHSrPpt9PsT/vWUekmieNVjQtvXxGO3yVxSZYu0tyE0vwqh7zpU/oa/WSLco0+rdB7WkRyltRXJIs2Ve1XvLc3uoWYnJNU/kQ/maZGs62sWQdjKD0tUgwhN/m6lFSCzrJNVaUD9fIn+slsOc3WnTo1u61P0pdHwq1AVmnrbZ1gpSxHr1TpQL38iX4yX7BWwwNL/iWtjLwSbpWySjNtvPdZvk2hD4lMUvkTvXv7LbhWW1mJD7srJa+EW6Ws0kwb732W++Eq7XurkejbVYHD68rLK+GGkFXybqy0+d5nFWaVvu5K9PXy/jJIuvJKuHlnlbxf3z2MnV2dqnzdleilWrJOeKGM5QshyYaws6modhN9sc+MFRmT5fSM9VMajo7C734Ht92W7Gmp7Qjh4qzLl0dXHps6NXo8dWr0OKQ5EipOc92ITFZIE6CEEEtRJsQpoXbnuuk60ZvZVGAIeMrdzzWz+cAdwEzgl8DF7v5aq+dolOj379/P8PAwr7zySlfxldGMGTPo6+tj+vTpeYdSTSFdhSyEJBvCzqai2k300xJ4rSuAh4B3xo+vB77m7neY2U3AZcA3Jvukw8PDHHzwwcybNw8zSyDMcnB3RkdHGR4eZv78+XmHU01z5zZObHlcgGUsmacxdWW7QigfSUtd1ejNrA/4M+Cb8WMDzgLuiVe5FfhQJ8/9yiuv0NvbqyRfx8zo7e0t15FO0WYQDW1Kw7SmrmyXLr8ZvG47Y/8W+CwwdrzaCzzv7q/Hj4eBIxv9oZmtNLMhMxsaGRlp+ORK8o2V6n0p4rXaqnBdvskIbcc3WUVraHSg40RvZucCe919U+3iBqs27ARw99XuPuDuA7Nnz+40DCm6okx4Xi/vVnRIirzjK2JDowPdtOjPBM4zsyeJOl/PImrhH2JmY7X/PuDpriJsVwX2yqWk+m45FHXH10lDo4C5puNE7+5/4+597j4PuBDY6O7LgZ8AF8SrrQDu7TrKiWS4V77mmmv46le/mvjz1lq3bh3HHnssxxxzDNddd12qr5U71XclT5NtaBT0CCCNE6b+GrjSzB4lqtnfksJrjFfUw/8G3njjDS6//HJ++MMfsn37dm6//Xa2b9+ed1jpKXp9V4ptsg2NguaaRBK9u//U3c+Nf37c3U9192Pc/aPu/moSr9FSiof/3/3udznxxBNZvHgxF1988bjf3XzzzZxyyiksXryYj3zkI7wcbwB33303ixYtYvHixSxduhSAbdu2ceqpp7JkyRJOPPFEduzY0fD17r//fo455hiOPvpoDjjgAC688ELuvTf9g6LcFLm+K8U32YZGQUuN5ZgCIaXD/23btrFq1So2btzIli1buOGGG8b9/sMf/jC/+MUv2LJlCwsXLuSWW6KDl2uvvZYf/ehHbNmyhbVr1wJw0003ccUVV7B582aGhobo6+tr+JpPPfUURx111JuP+/r6eOqpp7r6P4JX1PquFN9kGxoFLTWWI9GndPi/ceNGLrjgAmbNmgXAzJkzx/1+69atvPvd7+aEE05gcHCQbdu2AXDmmWdy6aWXcvPNN/PGG28AcMYZZ/DlL3+Z66+/np07d3LggQc2fM1GZyqXajilSGgm09AoaKmxHIk+pcN/d2+ZZC+99FJuvPFGHnzwQT7/+c+/eRLTTTfdxJe+9CV2797NkiVLGB0d5eMf/zhr167lwAMP5JxzzmHjxo0Nn7Ovr4/du3e/+Xh4eJgjjjiiq/9DRBJS0FJjORI9pHL4v2zZMu666y5GR0cB2Ldv37jfv/jii8yZM4f9+/czWNPr/thjj3Haaadx7bXXMmvWLHbv3s3jjz/O0Ucfzac+9SnOO+88HnjggYavecopp7Bjxw6eeOIJXnvtNe644w7OO++8rv8XEUlIAUuN5Un0KTj++OO5+uqrec973sPixYu58sorx/3+i1/8Iqeddhpnn302xx133JvLP/OZz3DCCSewaNEili5dyuLFi7nzzjtZtGgRS5Ys4eGHH+aSSy5p+JrTpk3jxhtv5JxzzmHhwoV87GMf4/jjj0/1/8xcAcchixRZsNMUP/TQQyxcuDCniMJX2PcnhNkWRUqi3dkr1aKXbBV0HLJIkSUxTbF0YHR0lGXLlr1t+YYNG+jt7c0hoowUdByySJEp0eekt7eXzZs35x1G9kKay12kIlS6kWwVdByySJEp0Uu2CjoOWaTIVLqR7C1frsQukqHStOg1NFtEpLFSJPosp4jOYj76T37ykxx22GEsWrQo1dcRkWooRaIv29DsSy+9lHXr1uUdhoiURCkSfZpDs7Oejx5g6dKlb5spU0SkU6VI9GlNEZ3HfPQiIkkrRaJPa2h2HvPRi4gkrRSJPq2h2XnMRy/SEQ07kxZKkeghnSmi85iPXmTSshx2JoXUcaI3sxlmdr+ZbTGzbWb2hXj5fDO7z8x2mNmdZnZAcuFmK4/56AEuuugizjjjDB555BH6+vrerP2LNFS2YWeSuI7no7eopnGQu79kZtOBfwWuAK4Evufud5jZTcAWd/9Gq+fSfPSTp/dH3jRlStSSr2cWHeJKaaU+H71HXoofTo9vDpwF3BMvvxX4UKevISJtSGvYmZRGV3PdmNlUYBNwDPB14DHgeXd/PV5lGDiyyd+uBFYCzK3gBlnZ+egleatWNb5ql2YElVhXid7d3wCWmNkhwD8AjWoJDWtD7r4aWA1R6abJOi1HvRRZN/PRh3D5RwnI2MiDq6+OzhKcOzdK8po4TmKJzF7p7s+b2U+B04FDzGxa3KrvA57u5DlnzJjB6Ogovb29pU32nXB3RkdHmTFjRt6hSEg0I6i00HGiN7PZwP44yR8IvA+4HvgJcAFwB7ACuLeT5+/r62N4eJiRkZFOQyytGTNm6MxaEWlbNy36OcCtcZ1+CnCXu//AzLYDd5jZl4BfAR2NDZw+fTrz58/vIjwREYEuEr27PwD8YYPljwOndhOUiIgkpzRnxoqISGNK9CIiJdfxmbGJBmE2Auzs8M9nAc8mGE7aihRvkWIFxZumIsUKxYq3m1j73X32RCsFkei7YWZD7ZwCHIoixVukWEHxpqlIsUKx4s0iVpVuRERKToleRKTkypDoV+cdwCQVKd4ixQqKN01FihWKFW/qsRa+Ri8iIq2VoUUvIiItKNGLiJRcoRJ9ES9faGZTzexXZvaD+HHIsT5pZg+a2WYzG4qXzTSz9XG8683s0LzjBDCzQ8zsHjN72MweMrMzAo712Pg9HbsZ62jcAAAD20lEQVT9xsw+HWq8AGb2V/F3bKuZ3R5/94Lcds3sijjObWb26XhZMO+tmX3LzPaa2daaZQ3js8jfmdmjZvaAmZ2URAyFSvTAq8BZ7r4YWAK838xOJ5o182vuvgB4DrgsxxjrXQE8VPM45FgB/tjdl9SM670K2BDHuyF+HIIbgHXufhywmOg9DjJWd38kfk+XACcDLxNdvyHIeM3sSOBTwIC7LwKmAhcS4LZrZouAvySaX2sxcK6ZLSCs9/Y7wPvrljWL7wPAgvi2Emh5Gda2uXshb0AP8EvgNKKzyqbFy88AfpR3fHEsffGHeBbwA8BCjTWO50lgVt2yR4A58c9zgEcCiPOdwBPEgwlCjrVB7H8C/CzkeImuCrcbmEk08eEPgHNC3HaBjwLfrHn8OeCzob23wDxga83jhvEB/wu4qNF63dyK1qIfK4VsBvYC65nE5Qtz8LdEG93YFZp7CTdWiK4G9mMz2xRf6hHgcHffAxDfH5ZbdG85GhgBvh2Xxb5pZgcRZqz1LgRuj38OMl53fwr4KrAL2AO8QHTJ0BC33a3AUjPrNbMe4E+Bowj0va3RLL6xneyYRN7nwiV6d3/Do0PgPqLDtbYvX5glMzsX2Ovum2oXN1g191hrnOnuJxEdPl5uZkvzDqiJacBJwDfc/Q+B3xJI2aOVuKZ9HnB33rG0EteLzwfmA0cABxFtE/Vy33bd/SGiktJ6YB2wBXi95R+FLZUcUbhEP8bdnwd+Ss3lC+NfdXz5woSdCZxnZk8SXW3rLKIWfoixAuDuT8f3e4lqyKcCz5jZHID4fm9+Eb5pGBh29/vix/cQJf4QY631AeCX7v5M/DjUeN8HPOHuI+6+H/ge8EcEuu26+y3ufpK7LwX2ATsI970d0yy+YaIjkjGJvM+FSvRmNtuiC5Fjb12+8CHeunwhdHH5wiS5+9+4e5+7zyM6XN/o7ssJMFYAMzvIzA4e+5molrwVWEsUJwQSr7v/GthtZsfGi5YB2wkw1joX8VbZBsKNdxdwupn1mJnx1vsb6rZ7WHw/F/gw0Xsc6ns7pll8a4FL4tE3pwMvjJV4upJ3Z8okOzROJLo84QNESei/xcuPBu4HHiU6LP6DvGOti/u9wA9CjjWOa0t82wZcHS/vJepQ3hHfz8w71jiuJcBQvC38I3BoqLHG8fYAo8C7apaFHO8XgIfj79ltwB8EvO3+X6Id0RZgWWjvLdGOZw+wn6jFflmz+IhKN18n6nt8kGjkU9cxaAoEEZGSK1TpRkREJk+JXkSk5JToRURKToleRKTklOhFREpOiV5EpOSU6EVESu7/Awr/X0lvaG3zAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x28285a37b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 1.读取数据\n",
    "def get_data(file_name):\n",
    "    with open(file_name) as file:\n",
    "        data_list = file.read().split('\\n')     # 使用readlines会包含回车符\n",
    "        # 使用map函数将子列表的每个元素转换为float\n",
    "        data_list = [list(map(float, row.split(','))) for row in data_list]\n",
    "        # 使用filter函数过滤数据，两类数据分开\n",
    "        label0 = np.array(list(filter(lambda x: x[-1]==0, data_list)))\n",
    "        label1 = np.array(list(filter(lambda x: x[-1]==1, data_list)))\n",
    "    x0, y0 = label0[:, 0], label0[:, 1]\n",
    "    x1, y1 = label1[:, 0], label1[:, 1]\n",
    "    plt.plot(x0, y0, 'ro', label='class_0')\n",
    "    plt.plot(x1, y1, 'bo', label='class_1')\n",
    "    plt.legend(loc='best')\n",
    "    plt.title('train data')\n",
    "    plt.show()\n",
    "    \n",
    "    x_train = np.concatenate((label0[:,:2], label1[:, :2]), axis=0)  # 按行拼接\n",
    "    y_train = np.array([0]*label0.shape[0] + [1]*label1.shape[0])    # 构建label\n",
    "    data_train = np.concatenate((x_train, y_train[:,np.newaxis]), axis=1)\n",
    "#     print(data_train)\n",
    "    return data_train        # 返回 numpy格式的训练数据\n",
    "mydata = get_data('data.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 产生训练模型的batch数据\n",
    "import random\n",
    "def get_batch(data, batch_size=100):                     # 生成数据的迭代器\n",
    "    \"\"\"data:特征+标签的numpy数据\n",
    "       batch_size:批次的大小\n",
    "       返回批次训练数据的生成器\"\"\"\n",
    "    train_data = data\n",
    "    order = list(range(len(train_data)))\n",
    "    random.shuffle(order)               # shuffle作用于多维数组会出现重复的行！！！\n",
    "    train_data = train_data[order]\n",
    "    index = 0\n",
    "#     print(type(data))\n",
    "    while True:\n",
    "        if index + batch_size < len(train_data):\n",
    "            total_data = train_data[index: index+batch_size] # 按照行访问数据\n",
    "            index += batch_size\n",
    "        else:\n",
    "            total_data = train_data[index:] \n",
    "            index = 0 \n",
    "            order = list(range(len(train_data)))  # 再次打乱顺序\n",
    "            random.shuffle(order)                 # shuffle作用于多维数组会出现重复的行！！！\n",
    "            train_data = train_data[order]\n",
    "        yield total_data[:, :2], total_data[:, -1:]\n",
    "g = get_batch(mydata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100, 2)\n",
      "(100, 1)\n"
     ]
    }
   ],
   "source": [
    "test_x, test_y = next(g)\n",
    "# print(test_x)\n",
    "print(test_x.shape)\n",
    "print(test_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2.定义模型、损失函数及优化方法\n",
    "class LogisticRegression(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.linear = nn.Linear(2, 1)   # 模型两个特征，一个输出\n",
    "        self.sigm   = nn.Sigmoid()\n",
    "    def forward(self, x):\n",
    "        x = self.linear(x)\n",
    "#         print('x:', x)\n",
    "        out = self.sigm(x)\n",
    "        return out\n",
    "# logistic_model = LogisticRegression().cuda() if torch.cuda.is_available() \\\n",
    "#                  else LogisticRegression()\n",
    "logistic_model = LogisticRegression().to(device)\n",
    "criterion = nn.BCELoss()\n",
    "optimizer = optim.SGD(logistic_model.parameters(), lr=1e-3, momentum=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**BCELoss()**\n",
    "\n",
    "定义: $loss(o,t)=-\\frac{1}{n}\\sum_i{(t_i \\cdot log(o_i)+(1-t_i) \\cdot log(1-o_i))}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------\n",
      "epoch 1000\n",
      "loss is 0.5704\n",
      "acc is 0.6300\n",
      "----------\n",
      "epoch 2000\n",
      "loss is 0.5350\n",
      "acc is 0.6600\n",
      "----------\n",
      "epoch 3000\n",
      "loss is 0.5049\n",
      "acc is 0.7000\n",
      "----------\n",
      "epoch 4000\n",
      "loss is 0.4793\n",
      "acc is 0.7700\n",
      "----------\n",
      "epoch 5000\n",
      "loss is 0.4574\n",
      "acc is 0.8100\n",
      "----------\n",
      "epoch 6000\n",
      "loss is 0.4384\n",
      "acc is 0.8300\n",
      "----------\n",
      "epoch 7000\n",
      "loss is 0.4219\n",
      "acc is 0.8600\n",
      "----------\n",
      "epoch 8000\n",
      "loss is 0.4074\n",
      "acc is 0.8900\n",
      "----------\n",
      "epoch 9000\n",
      "loss is 0.3947\n",
      "acc is 0.9000\n",
      "----------\n",
      "epoch 10000\n",
      "loss is 0.3833\n",
      "acc is 0.9100\n"
     ]
    }
   ],
   "source": [
    "# 3.训练网络\n",
    "num_batches = 10000#10000                      # ！！！注意这里的batch数据迭代的次数，不是epochs \n",
    "gen = get_batch(mydata, batch_size=100)   # batch_size=100访问所有数据,num_batches等同于epochs\n",
    "for epoch in range(num_batches):\n",
    "    x, y = next(gen) \n",
    "    x, y = map(torch.Tensor, (x,y))       # 将x，y转换为张量   \n",
    "    x_train, y_train = x.to(device), y.to(device)\n",
    "#     x_train, y_train = (Variable(x).cuda(), Variable(y).cuda()) \\\n",
    "#                        if torch.cuda.is_available() \\\n",
    "#                        else (Variable(x), Variable(y))  # 转换为Variable\n",
    "\n",
    "    # 前向传播\n",
    "    out = logistic_model(x_train)         # 计算输出\n",
    "    loss = criterion(out, y_train)        # 计算损失函数\n",
    "    print_loss = loss.data                # 获取损失函数的值\n",
    "#     print('=====', print_loss)\n",
    "    mask = out.ge(0.5).float()            # 大于0.5的设置为1，否则设置为0\n",
    "    correct = (mask == y_train).sum()     # 统计预测正确的个数，这里得到整型tensor\n",
    "    # 必须转换为numpy运算才行\n",
    "    \n",
    "    acc = correct.data.float() / y_train.shape[0]   # 计算准确率size(0)\n",
    "#     print(acc)\n",
    "    # 反向传播\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    # 输出验证的结果\n",
    "    if (epoch + 1) % 1000 == 0:\n",
    "        print('-'*10)\n",
    "        print('epoch {}'.format(epoch+1))\n",
    "        print('loss is {:.4f}'.format(print_loss))\n",
    "        print('acc is {:.4f}'.format(acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd4FNX6wPHvSagBpAQQJJDQBOkIKAgCggp6BUXRq6CC+rtc0Ss2FCyXYrsWUFFUxK6giAXFiAjSURADhF5CSwApIXQChGTf3x+7wRBTNtkyM7vv53n22exkdubd2dl3zpxz5owREZRSSoWuCKsDUEopFVia6JVSKsRpoldKqRCniV4ppUKcJnqllApxmuiVUirEaaJXYccYM8EY818/LWuUMWaSP5alVKBooleOYozZYYy50pdliMi9IvKsv2LyljHmY2PMc8Fer1Ka6FVIMcaUsDoGpexGE71yDGPMZ0Ad4AdjzHFjzOPGmDhjjBhj7jHGpABzPfN+ZYzZa4w5YoxZaIxpmmM5Z0vWxpiuxphdxphHjTH7jTF7jDF3FRBDXWPMAmPMMWPMbKBqrv/nuV5jzCCgP/C4J/YfPNOHG2O2epa33hjTx79bTSlN9MpBROQOIAXoJSLlReTlHP/uAlwE9PC8/gloCFQHVgCTC1h0DaAiUAu4B3jLGFM5n3k/B5bjTvDPAgNy/T/P9YrIRM/fL3ti7+WZfytwuWf9o4FJxpiaBcSqVJFpolehYpSInBCRkwAi8qGIHBOR08AooKUxpmI+7z0DPCMiZ0RkBnAcaJR7JmNMHaAd8F8ROS0iC4Efcs5TxPUiIl+JyJ8i4hKRL4Ek4JIifnalCqSJXoWKndl/GGMijTEveqpEjgI7PP+qmuc7IU1EMnO8TgfK5zHfBcAhETmRY1qyD+vFGHOnMSbRGHPYGHMYaFbQ/EoVhyZ65TT5Dbeac3o/4HrgStxVInGe6cbHde8BKhtjyuWYVqcI6z0ndmNMLPAe8B8gWkQqAWv9EKdS59BEr5xmH1CvkHkqAKeBNCAKeMEfKxaRZCABGG2MKWWM6QT0yjFLYevNHXs53Mk/FcDTCNzMH7EqlZMmeuU0/wOe9lR1DM1nnk9xV6nsBtYDS/24/n7ApcBBYKRnXd6u9wOgiSf270RkPTAWWIL7INAc+NWPsSoFgNEbjyilVGjTEr1SSoU4TfRKKRXiNNErpVSI00SvlFIhzhYDQFWtWlXi4uKsDkMppRxl+fLlB0SkWmHz2SLRx8XFkZCQYHUYSinlKMaY5MLn0qobpZQKeZrolVIqxGmiV0qpEFdoojfGfOi5IcPaHNOqGGNmG2OSPM+VPdONMeYNY8wWY8xqY8zFgQxeKaVU4bwp0X8M9Mw1bTgwR0QaAnM8rwGuwX3ThYbAIOAd/4SplFKquApN9J6bKxzMNfl64BPP358AN+SY/qm4LQUq2fpuOZMnQ1wcRES4nycXdBMipZRypuLW0Z8vInsAPM/VPdNrkeMGEMAuz7S/McYMMsYkGGMSUlNTixmGDyZPhkGDIDkZRNzPgwZpsldKhRx/N8bmdcOEPIfHFJGJItJWRNpWq1Zof3//e+opSE8/d1p6unu6UkqFkOIm+n3ZVTKe5/2e6buA2jnmiwH+LH54AZSSUrTpfqK1ReFJv3dlpeIm+unAAM/fA4Dvc0y/09P7pj1wJLuKx3bq1CnadD/Q2qLwpN+7spo33Su/wH0HnEbGmF3GmHuAF4GrjDFJwFWe1wAzgG3AFtz3wrwvIFH7w/PPQ1TUudOiotzTA0Rri4Indwn6vvusK1E74XvXM44QJyKWP9q0aSOWmDRJJDZWxBj386RJAV2dMSLuMt25D2MCutqwM2mSSFRU3ts6+xEVFfCv+6zifu/B2j3z2l7B3D6q+IAE8SLHWp7kJViJPshJPS+xsXn/4GNjgx5KSMtvO1u13YvzvQcz+ep+6VzeJvrwGALBJpWkFtQWhSVv29MD3O5+VnG+92BW91jUL0EFUXgkeptUkvbvDxMnQmwsGON+njjRPV35j7ft6QFsdz9Hcb73YCZfC/olqCALj0RvoyJL//6wYwe4XO5nTfL+l1cJOrdgn0kV9XsPZvLVM83QFx6JXossYSWvEvTgwc46kwpm8tUzzdBn3PX51mrbtq0E9A5T2XX0OatvoqJ0b1a2Nnmyu3YxJcVdJnn+ed1d1bmMMctFpG1h84VHiV6LLMqGCuu7rtV8yl9scc/YoOjfX38pyjZyn2RmdwQD3U2V/4VHiV4pm7FJRzCv6FWzzhc+JXqlbMRGHcEKpGceoUFL9EpZwCkdwfI783jwQWviUcWjiV4pCzil73p+ZxhpaVqF4ySa6JWyQKA6gvm7Pr2gMww7tieovDk60W/ce5QXf9rIidOZVoeiVJH5u/tkIIZ0KugMw27tCSp/jk70izYfYMKCrXQbO5/vE3djh4u/1F+0t0ZwZG/n22/3f0+e/v0hOjrv/9mtPUHlz9GJ/l+d6/HN4MuoXqEMD05J5J/vLmX9n0etDkthmwFDQ17O7ZwfX0ve48Y5oz2hIGFf6PBmLONAP3wdjz4zyyWf/54srUb/LHWHx8t/v1sjh06c9mmZocSKofh1jPPg8GbsfX9scxvczqHYQvnGKng5Hn1IjXVzJP0Mr87exGdLk6lYtiSP9WjMP9vVJjLC+CFKZ7JqmJ+ICPdPKjdj3HXSyj/y287ZdEgndwk+rzOe2Fh324iTheVYNxWjSjL6+mbEP3A5DatX4Mlpa7j+rcUsTz5kdWiWseoKTKf0E3e6grZndDSULQt33BF+1RU5q2ryq9YKp8bkkEr02ZpccB5f/rs9425tReqx09z0zm88MjWR/cdOWR1aofxdl2jVFZhO6SfudPlt58GD4eRJd3/3cGsjyd0+lJ+wKnR4U78T6Ecg7xl7/NQZefGnDdLgyR+l6YiZ8t7CrZKRmRWw9fkiEHWJVtaVO7le10ny2s7h3EbiTbtFIOvog7nfozcHP9e21OMy4MPfJXZYvHQfO18WbU4N+DqLKhA/zlBuiMpJDyrnMibvfckYqyMLvPw+e/bnD+T+Eezfmyb6PLhcLpm9bq9c/tJciR0WL//+NEFS0k4EZd3eCNSPM9SToB0PZlZt84JK8+Feog/GZw/2ujXRF+BkRqa8OWezNHp6hlz41Ax5bfYmOZmRGdQY8hLOp9u+sNt2s+rAk9d67XTwCxYrD/zBPpPSRO+F3YfS5b7JyyV2WLx0fHGO/LRmj7hcLktiEbFnydQJ7FZNEcwDT84zh8jI/JN8KJ7JFcSqMyot0dsw0Wf7dUuqXP3qAokdFi+3v79UkvYdsyyWUK9mCQS7leiDdeAprARv9QEvHNm1jj6kLpjyRWaWi8+WJvPq7M2czMji7k51eaBbAyqUKWlpXKpwdrv3e7Au0MlvPYFerypYMG/qHpYXTPmiRGQEd3Wsy7yhXbnx4lpMXLiNbmMX8O2KXdjhYKjyZ7d7vwfrGgJvroXQaxeCz443ddcSfT4Sdx5m5PdrWbXrCG1iKzO6d1Oa1apodVjKIYJRqsuvRB8Z6U4ygS5NKut5W6LXRF8Al0v4avlOXp65iYPpGfS7pA5Dr25E5XKlrA5NKdtVWang06obP4iIMPyzXR3mDu3KgA5xTPljJ1eMnc9nS5PJcll/gFThzW5VVsq+tERfBJv2HmPk9LUs3XaQJjXPY/T1TWkXV8XqsJRSYUpL9AHQqEYFvvhXe8b3a82h9AxunrCEh6asZN9R+w+WpgoX9jenUCFLS/TFlJ6RydvztjJx4TZKRhqGdG/IXR3rUqqEHjudSOu7lRMFpTHWGPMw8H+AAGuAu4CawBSgCrACuENEMgpajhMTfbYdB07wbPx65mzcT71q5RjZqyldLqxmdViqiEL55hQqdAW86sYYUwsYArQVkWZAJHAr8BLwmog0BA4B9xR3HU4QV7UcHwxsx0cD2+FyCQM+XMa/Pk1g58H0wt+sbMOqcfuVCgZf6xlKAGWNMSWAKGAP0A342vP/T4AbfFyHI1zRuDo/P9yZx3s24tctB+j+6gJenbWJkxlZVoemvKB3xFKhrNiJXkR2A2OAFNwJ/giwHDgsIpme2XYBtfJ6vzFmkDEmwRiTkJqaWtwwbKV0iUju69qAOY92oWfTGrwxdwtXvrqAGWv2eHd1rbYGWkbviKVCmS9VN5WB64G6wAVAOeCaPGbNM8OJyEQRaSsibatVC6067ZoVy/LGba2ZMqg9FcqU4L7JK7j9g99J2ncs/zflvv9ZON37zQbs0iddj/UqEHypurkS2C4iqSJyBvgWuAyo5KnKAYgB/vQxxuAIwC+sfb1o4h/oxOjeTVmz6wjXjFvEs/HrOXrqzN9ntuou3so29FivAqXYvW6MMZcCHwLtgJPAx0AC0Bn4RkSmGGMmAKtF5O2ClmV5r5sg9K1LO36aMbM2MeWPnUSXK83waxpzY+taREQY9wwREe5fd27GuAcuUQFlh+6V2vNHFVWwuleOBv4JZAIrcXe1rMVf3StXAreLyOmClmN5og/iL2z1rsOMnL6OlSmHaV2nEs/0bkbzmIr6K7eYHTa/HuvtIZjDDPtKBzUriiD/wlwu4ZsVu3hp5kbSTmRwa7vaPHl4JRUeuE+v2LGIHZKsHQ42xeWk5FgQO5zZFYUOgVAUgexbl0fdf0SE4ea2tZk7tCt3d6zL1IRddEypwcLHnkfq1LHNCFV2bBgMVEx26F7p1J4/odS2ELJNZd7chirQD6tvJRiw+395udzNe49Kv/eWSOyweOnx2gJZuvWAb+v1AzvevzaQMdnl81p9K8nirN9ut3L0hd3uP1wY9J6xRTBpkkh09F/fanS0f35hRfgFuFwumbH6T7nsf3Mkdli8PPD5Ctlz+KTvMRSTHX+8gY7J6iRrteIe7JyWHAtix/2+IN4meq2jD2SlXDEqfk9mZPHOgq1MWLCVEhGG/3RrwD2d6lK6RKRvsRSRHeqsc7NjTKGkuG0ETm5byE3r6ENVICvlilHxW7ZUJI9cdSG/PNyFjg2q8vLMTfR8fRHzNu73PZ4isEOdtbfr1mEK/CO/cX2SkwtuE3Fq20Je7HLhnN95U+wP9MPSqptAnnf6oeJ3/qb9csUr8yR2WLzc/dEy2XHguO9xecEuddZ2jymU5Fdt4c32DvdqL6ugdfReckDF7+kzWTJh/hZp8t+fpOGTM+TlmRvkxOkz/omvAHb88RYUkx3jdZK8DqROqq8OR5rovVVYMdFG2WPvkZPy0JSVEjssXtq/8Iv8sGq3uFwuy+Kxk7y+xlKl3O3qNvjqHCPn7p5fondiI2uo0kRfFPklc5vWFSzbnibXvL5QYofFy63vLpGNe45aGo8d+FLtoPLmtB4oTuKv8qO3iV573RTExt0JslzC58tSGDtrE8dOZXJH+1gevupCKpYtaWlcVsmvR05uNvjqHMNpPVCcwp/bVYdA8AcH9Oc7dCKDMbM28fmyFKpElWJYz8b0bRPz12BpYSK/Y3JuNvrqHCFUhjawE3+WH7V7pT/4uz9fAK7fr1yuFM/3ac4P/+lEXNVyPP7Navq88xuJOw/7vGwnyauLX160K2bR9O/vTj4ul/tZk7zvrLhtpSb6gvizg3CABwRpVqsiX9/bgVdvacmfh09yw1u/8vjXqzhwvMCBQ0NG7v7P0dFQMlctllP7dvuTHccvCjeWXA/iTUV+oB+WN8YWxF+tJkFs2Tp6MkOe/3G91H/iR2k2cqZ8sGibnMnM8vt67C73Vzd4sG06UFnCpn0Lwo4/vwe0143NWDAgSNK+Y3L7+0sldli8XPXqfPl1S6rflm2jXqdeCcUkV9TvINR60ThtH8wp2L1uLE/yEi6J3qJfmcvlkplr90jHF92Dpd03ebnsPpTu0zKdmDRDMckV9TsoqG+8ExOl0/bBQNBEbzcW75knMzLl9dmb5cKnZkjjp3+SN+dslpMZmcValhOTZiiNsChSvO+gsGsNnJQo7bwPBvPqbU30dmSDc82UtBPy708TJHZYvFz+0lz5Zf3eIi/DiUnTzomhOIrzHXgzxIFTtodd98GCynOBKOt5m+i1H32YWpx0gJHT17I19QRXNKrGiF5NqVu1nFfvtfF1ZPkKtYt/ivsdZPeLz++aA6dcZ2DXfbCguMD/MWs/elWgTg2rMvOhzjz9j4v4Y8chery2kJdmbuTE6cxC3+vEYWlDbfjZ4n4H2f3isxNPbk65zsCu+2BBfeSt6D9/ljfF/kA/wqbqxqb2HT0pj3yZKLHD4uXS53+R71buKnSwNBvUQoU9X76DUGjMtOM+WFAVYSCqD9E6+hAXgL08YcdB+ccb7sHSbp7wm6z/84jPywwHdkw43nBq3HZm1zp6y5O8aKIvugAWxzKzXDJ5abK0Gv2z1B0eLyO+WyOHT2T4IejQFAol41Bgp4OWHXvdaGOsEwWhJepwegavzt7MpKXJVIoqxWM9GnFL29pEhtlgaYWxa6NgOAlUQ7sTBnTTxlgrBXpAkSC06lSKKsUz1zcj/oHLaVCtPE98u4Yb3vqVFSmHfF52KI23YmkDmwICc9vnAA9NFXRaove3YPTjC3IxUkSYvupPXpixgX1HT3PTxTEMu6YR1SuUKfKytJuj8rdAjCbulO9VS/RWCUTxIrcg9y0zxnB9q1rMfbQr93apz/RVu+k+ZgHvL9rGmayi/ZKCsXmCya7d/MJJIEaDDLUzNU30/haMPcSiTuHlSpdg+DWN+fmhzrSJq8xzP27gmnGL+HXLAa+XEWo/oFDrn+9EgTjYWjKUcCB502Ib6EdI9boJtWvt8+FyuWT1S2/JnkrVJQsjB6JryIF3Pyz0fWGyeVSQ+bs3i1N6U6HdKy3ilD3EV3l8zvSSpWXG8DEFDpYWLptHOZ+dumzmRxO9lZywh/gqn6L5zvOqSccX58jPa/fke3VtOGwepYLB20SvvW5U8eTT1UGMocfYeWzed5zOF1ZjZK8m1K9W3oIAlQp92utGBVY+rVKmTh1+HHI5I65rwsrkQ/R8fSH/m7GB414MlqaUCgxN9Kp48urqYAxcey0lIyO4u1Nd5g7tyg2tavHuwm10GzOfaSt3YYczSH8IpYu+VPBYtd9oos+mv9yi6d8fBgxwJ/dsIvDJJ2e3XbUKpXnl5pZMu+8yalYsw8NfruLmCUtYu/uIRUH7R6hdNamCw9L9xpuK/PweQCXga2AjsAHoAFQBZgNJnufKhS3H8sZY7Qryl6K0lBahr2RWlkumLEuWi5+ZJXWHx8tT01bLweOnA/QhAku7iKrisHKYYp8aY40xnwCLROR9Y0wpIAp4EjgoIi8aY4Z7Ev2wgpZjeWOsU653DrSijk9QjGvPj5w8w2uzN/PZ0mQqlCnB0Ksbcdsldc4ZLM3ug0kF4pJ7FfoCsd942xhb7ERvjDkPWAXUkxwLMcZsArqKyB5jTE1gvog0KmhZlid6/eW6FfWA58MBcuPeo4z8fh2/bz9I0wvOY3TvprSNq+KIsXC0XKCKIxD7TTB63dQDUoGPjDErjTHvG2PKAeeLyB4Az3P1fAIcZIxJMMYkpKam+hCGH4Tc9c7FVNTxCXy49rxxjfOYMqg9b97WmoMnMug7YQkPf5nI8Cdcth8LR8e3UcVh6X7jTf1OXg+gLZAJXOp5PQ54Fjica75DhS1L6+htojiViH64+unE6TPy8swN0vDJGQKuPEMwppifKUD0oi9VHFbdeMSXRF8D2JHj9eXAj8AmoKZnWk1gU2HLsjzRi+gvV8TyA9721ONSLvqUNnQq5SVvE32xq25EZC+w0xiTXf/eHVgPTAcGeKYNAL4v7jqCqn9/d0WZy+V+tkuFcDBZPBRjXNVyvDuuNKXLnNsuUrasaLVIiNJezcFRwsf3PwBM9vS42Qbchbvef6ox5h4gBbjZx3WoYOrf39KDnHvVETz5pJCyE0qcd4qKXTexr3oUJzPqU7ZUpGWxKf/K3fCe3a8cwrOcFUg+XTAlIoki0lZEWojIDSJySETSRKS7iDT0PB/0V7AqDyFYJOrfH5KTDeIypCRD338Kb8xJ4spXFzBz7Z7sqkLlEPntoqF2Exo700HNnMwJfRH9ZMnWNEb/sI6Ne4/RqUFVRvVuQoPqFawOSxWioF30jju0V7OvAt6P3p800RdTmHXozsxyMWlpMq/O3kx6RhYDL4vjwSsbUqFMSatDU/koaBeFsNp9A0JHrwwHoXZfvkKUiIxgYMe6zBvalb5tYvjg1+1cMWYBXy/fhctlfYFF/V1Bu6hejxA8muidLEwv9IouX5oXb2rBd/d1JKZyWYZ+tYq+E35z/GBpoaigXVTvtxs8muidLMyLRC1rV+LbwZfxSt8WpBxMp9f4xTzx7RoOnsiwOjTlUdguqr2ag0MTvZNpkYiICMPNbWszd2hX7rqsLlMTdnLFmPl8umQHmVnaomc13UXtQRtjVUjZvO8Yo6av47etaTSuUYFnrm/GJXWrWB2WUgGhjbEqLF14fgUm/9+lvN3/Yo6ePMMt7y5hyBcr2XvklCXxhOBlDsqBNNGr4AhixjPGcG3zmsx5tCtDujVg5rq9dBs7n9lPv4bExgYt6+qdqJRteDMgTqAfthjULJwFekA3iwdLSz5wQiYOekZOlCgd1Bj0TlQq0AjGHab8RevoLRSMq2vtcGGXBTHo/WxUoGkdvfJOMAYcscOFXfmsS1JSOJmRFZBVhullDsqGNNGHu2AkYasz3uTJ7uJ1HnZXqEr3sfP5cbX/B0sL88sclI1oog93BSVhfzWgWpnxsqumsvIotUdFcWr0c1SMKsX9n6+g33u/s3nfMb+tWvuQK9vwpiI/0I9iN8bqXaF8l19D6eDB/m1Ateq7yq9FNDLybAyZWS759Lft0mLUz1LviR9l1PS1cjg9IzjxKeUDQr4xNoyG6A24yZPddfIpKe6S/PPPu1/n1XgZHQ0HDgQ/xuIqQovowRMZjJm1iS+WpRBdrhSP92hM3zYxRESYIAWrVNGEfmOs3rXAf/IacCS/Ovq0NGd1BC9C+0CVcqV4oU9zfvhPJ2Kjy/H4N6vp885vrNp5OMBBhia9WMw+nJvo7dCTI5QV1FDqpINpMdoHmtWqyNf3duDVW1qy+9BJbnj7V4Z9vZoDx08HONjQoReL2YtzE73VPTlCXUENpU46mBazRdQYw40XxzBvaBf+r1NdvlmxiyvGzOejX7c7drC0YJaw9YTbZrypyA/0o1iNsYG82lIbed2io/XSTo+kfUfl9veXSuyweLn61QXy25YDVodUJMG+ONmYvHcdYwKzvnCFl42xzi3RB6rvmp5z/mXcuL9Xe5QsCcePh13Fa4PqFfj07kuYcHsbjp/O5Lb3lnL/5yv48/DJc2e0qGK6sNUGu4StJ9w2483RINAPW411owOUnCvn2U10tEipUsErFtrUyYxMeW32JrnwqRnS+OmfZPzcJDl1JtOyMX28WW2wS9gWD28UNvCyRG95khe7JXo958zbpEnuvud6EDwrJe2EDPr0D4kdFi+dX54r6TVjLNk+3pRNrCi/hEoNqJ0/hyb64tIS/d/lVTzTg+BZCzfvl25j5kkW1hQSvCmbhEMJOxAJ2e7bTRN9cdn9m7VCfgc/PQiedfpMlhw9/wJLto+3ZZPiJkI7l2izBepna/dynyb6osi9Jw8ebP89O5jyKzLqQfBckyaJq+y52eZMmbLiskEdvR2X7U+BSsh2r8kNn0Tva3HDKXuylbwYL0Z5ePZHlzGyt/L58sB1j8otE36T9X8eCcZq/V42sXuJNlugErLdP394JPqiJOn8fgl2/ybtQA+GxZKZ5ZJJS3dIy9E/S93h8TLiuzVy+ISzBkvzZwINZBVQQbWLvqzL7rt+eCT6olRO5vdt2f3czC6cUFFrlUK2zaETp+XpaWuk7vB4af3MLPni92TJynJZEmpR+ascFOiEWVh/AScOvOqN8Ej03ibpgvZWLdGHDit+kUXIYGt3H5a+7/wqscPipdebi2RF8sHAx+cjrz9eIds+GD+z7BAKKtmHmvBI9N7uPQUdEOx+bqa8Y9X3WMQM5nK5ZNqKXdLuudkSOyxeHp2aKPuPngpsjD4q9PjpxbYP5olzOJ2kh0ei9/bHXdiP0c7nZso7Vp2ZFTOrHDt1Rl6YsV4aPPmjNBsxU95buFUyMrOKF4PV+68X2z6YX084naSHR6IX8W4n11J76AtmMS7nPufj1cJb9h+TOz/4XWKHxcuVY+fLr0mpRY/F6n3bi20fzDDtsEmCJXwSvbesLvWowApWMa6wVr9iZBWXyyWz1u2VTi/Nkdhh8TJ4UoLsOpTu3ZvtUHz1MoZg/gTD5ecetEQPRAIrgXjP67rA70AS8CVQqrBlWH7BlHK+YBXjCrqmwMescjIjU8b9slkaPT1DGj09Q8b9sllOZmQW/CY7VEgPHpx3DIMHBy+GMOVtovfHMMUPAhtyvH4JeE1EGgKHgHv8sA6lChaoYatzy++mKy7XubdiLIYyJSMZ0r0hvzzShW6Nq/Pq7M1c9doCZq3bm12o+js7jAc8Y0bRpqug8ynRG2NigH8A73teG6Ab8LVnlk+AG3xZhwoD/hrDPa973/pbEBJrTOUo3u7fhsn/dyllSkQy6LPlDPjoD7amHv/7zMW4VaLf6W09bc/XEv3rwONA9r3VooHDIpLpeb0LqJXXG40xg4wxCcaYhNTUVB/DUI7ltBu9BDGxdmxQlRkPXs5/r2vCyuRD9Hx9If/7aQPHT2f+NVOwzmQKYoezCl+Ew13MvanfyesBXAe87fm7KxAPVAO25JinNrCmsGVpHX0Ys0NjYlFZ0NK3/+gpeXRqosQOi5d2z82WaSt2ictlk6trndzNpTix26ill0A3xgL/w11i3wHsBdKBycABoIRnng7Az4UtSxN9GLNDY6KDLE8+KL3eXCSxw+Kl7zu/ytrdh60Oyc1Gya9IilrQsNlBzdtEbyS/Rp4iMMZ0BYaKyHXGmK+Ab0RkijFmArBaRN4u6P1t27aVhIQEn+NQDhQX566uyS021l3Prv7G5RKmJuzk5Z83cTg9g36X1mHo1Y2oFFXK6tCcJyLCna5zM8bd1pObzfZSQOLHAAAW50lEQVRXY8xyEWlb2HyBuDn4MOARY8wW3HX2HwRgHSpU2KEx0WEiIgy3XlKHeY925c4OcXz+ewpXjJnP5N+TyXL5XnALK0VtX3Bow7NfEr2IzBeR6zx/bxORS0SkgYjcLCKn/bEOFaLs0JjoUBWjSjKqd1N+HHI5Dc+vwFPT1nL9W4tZnnzQ6tCco6gFDYc2PAeiRK9U0QSjW2QIu6jmeXw5qD1v3NaaA8cyuOmdJTzyZSL7j56yOjT7K2pBw6FnoJroVfCFQ3e2IDPG0LvlBcx5tAv3da1P/Oo9dBu7gPcWbiMjM4+6ZvWXohQ0HHoG6pfGWF9pY2wYye43n57+17SoKEf8WJxk+4ETPBu/nrkb91O/WjlG9W7K5Q2rWR2W8jNvG2M10avgslmvhVA3Z8M+nolfT3JaOj2ans/T/2hC7SpRhb9ROYImemVPRe3Opnx26kwWHyzezvi5W3CJcG+X+gzuWp8yJSOtDk35yMrulUrlz6G9FpysTMlI7r+iAXMe7cJVTc5n3Jwkuo9dwMy1BQyWpkKKJnoVXA7ttRAKLqhUlvH9LuaLf7WnfOkS3DtpOXd+uIwt+/MYLE2FFE30Krgc2mshlHSoH82PQzoxslcTEncepufrC3n+x/UcO3XG6tBUgGgdvVJh7MDx07wycxNTl++kavnSDO/ZmD6taxERYawOTXlB6+iVUoWqWr40L/VtwXf3deSCSmV59KtV9J3wG2t3H7E6NOVHmuiVUrSsXYlpgy/j5b4tSDmYTq/xi3ly2hoOnciwOjTlB5rolVKAe7C0W9rWZs6jXbnrsrp8+cdOuo6Zz2dLduhgaQ6niV6pUODHYSUqli3JiF5NmDHkcprUPI//fr+O695czLLtOliaU2miV8rpAnQ7xkY1KvD5vy7lrX4XcyQ9g1veXcKDU1ayTwdLcxztdaOU0wVhWIn0jEzemb+Vdxduo2SE4YHuDbm7Y11KldCyopV0CASlwkUQh5VITnMPlvbLhv3Uq1qOEb2a0LVRdb+uQ3lPu1cqFS6COKxEbHQ53h/Qjo/uaocAAz/6g//7JIGUtPRC36uso4leKaezYFiJKxpVZ+ZDlzOsZ2N+23qAK19bwNhZmziZkRWwdari00SvlNNZNKxE6RKRDO5an7mPduWaZjV4c+4Wuo+dz4w1e3SwNJuxbR39mTNn2LVrF6dOaQt/oJUpU4aYmBhKlixpdSjKwZZtP8iI79eyce8xLqsfzajeTbnw/ApWhxXSHN8Yu337dipUqEB0dDTG6LgbgSIipKWlcezYMerWrWt1OMrhMrNcfL4shbGzNnP8dCYDOsTx0FUNOa+MFiICwfGNsadOndIkHwTGGKKjo/XMSflFicgI7uwQx7yhXbmlbW0++m073cbMZ2rCTlx6da1lbJvoAU3yQaLbWflblXKl+N+NzZl+fyfqVIni8a9Xc+M7v7Fq52GrQwtLtk70Silnax5Tka/vvYyxN7dk16GT3PD2rwz/ZjVpx09bHVpY0URfgMjISFq1anX2sWPHDhISEhgyZAgA8+fP57fffjs7/3fffcf69evPvh4xYgS//PKLX2KJi4vjwIED50ybPn06L774ol+Wr1SgREQYbmoTw7yhXbinY12+Xr6LK8bM5+Nft5OZpfcJDgbbNsZu2LCBiy66yKKI3MqXL8/x4/nfZm3UqFGUL1+eoUOHAjBw4ECuu+46+vbt6/dY4uLiSEhIoGrVqn5fNthje6vwsGX/MUZNX8/iLQdoXKMCo3o3pX29aKvDciTH97rJmXhG/7CO9X8e9es6m1xwHiN7NS1wnrwS/fz58xkzZgzjx4+nffv2REZGUq1aNcaNG0efPn2oWLEiFStW5JtvvuHZZ589m/jj4uIYMGAAP/zwA2fOnOGrr76icePGpKam0q9fP9LS0mjXrh0zZ85k+fLlf0voeSX6jz/+mISEBMaPH8/AgQM577zzSEhIYO/evbz88stnDzivvPIKU6dO5fTp0/Tp04fRo0f/7bNqolfBJCL8vG4vz8ZvYPfhk/RqeQFPXtuYmhXLWh2aozi+140dnDx58my1TZ8+fc75X1xcHPfeey8PP/wwiYmJdOnShd69e/PKK6+QmJhI/fr1/7a8qlWrsmLFCgYPHsyYMWMAGD16NN26dWPFihX06dOHlJSUYse7Z88eFi9eTHx8PMOHDwdg1qxZJCUlsWzZMhITE1m+fDkLFy4s9jqU8gdjDD2b1eSXR7owpHtDfl63l25jFvDWvC2cztSra/2thNUBeKOwkneglC1blsTERL8t78YbbwSgTZs2fPvttwAsXryYadOmAdCzZ08qV65c7OXfcMMNRERE0KRJE/bt2we4E/2sWbNo3bo1AMePHycpKYnOnTv78lGU8ouypSJ55KoLublNDM/Gr+eVnzfxVcJORvRqQrfG51sdXshwRKIPFaVLlwbcjbyZmZkAfr1UPHv5OZcrIjzxxBP8+9//9tt6lPK32lWimHhnWxZuTmXUD+u4++MEujWuzojrmhBXtZzV4TmeVt34oEKFChw7dizf197o1KkTU6dOBdyl70OHDvk1xh49evDhhx+ebWvYvXs3+/fv9+s6lPKXzhdWY+aDnXny2sb8vi2Nq19byCs/byQ9I9Pq0BxNE70PevXqxbRp02jVqhWLFi3i1ltv5ZVXXqF169Zs3brVq2WMHDmSWbNmcfHFF/PTTz9Rs2ZNKlTIe3yQFi1aEBMTQ0xMDI888ohXy7/66qvp168fHTp0oHnz5vTt27fIByOlgqlUiQgGda7PvKFdua5FTd6at5XuYxfww6o/dbC0YnJEr5tQdvr0aSIjIylRogRLlixh8ODBfm0X8Fa4bG/lPAk7DjLi+3Ws33OU9vWqMKp3UxrXOM/qsGzB2143WkdvsZSUFG655RZcLhelSpXivffeszokpWylbVwVfnigE18sS2HMrE38443F3NE+loevupCKZXWwNG9oordYw4YNWblypdVhKGVrkRGG29vH8o/mNRk7exOfLtnB9FV/8niPRtzStjYRETpeU0GKXUdvjKltjJlnjNlgjFlnjHnQM72KMWa2MSbJ81z8/oJKKZVD5XKleO6G5kz/TyfqVS3H8G/X0OftX0nUwdIK5EtjbCbwqIhcBLQH7jfGNAGGA3NEpCEwx/NaKaX8plmtinx1bwde/2cr9hw5xQ1v/cpjX60i9ZgOlpaXYid6EdkjIis8fx8DNgC1gOuBTzyzfQLc4GuQSimVmzGGG1rXYu7Qrvy7cz2+S9xNtzHz+WDxds7oYGnn8Ev3SmNMHNAa+B04X0T2gPtgAFTP5z2DjDEJxpiE1NRUf4ShlApD5UuX4IlrL2LmQ51pHVuZZ+PX8483FvHblgOFvzlM+JzojTHlgW+Ah0TE65HHRGSiiLQVkbbVqlXzNQyYPBni4iAiwv08ebLvy1RKOUb9auX55K52TLyjDSfPZNHv/d+5f/IKdh8+aXVolvMp0RtjSuJO8pNF5FvP5H3GmJqe/9cEAn8Z5uTJMGgQJCeDiPt50KCAJPtRo0adHZAsUGbOnEmjRo1o0KCBjjevVBEYY7i6aQ1mP9yFh6+8kF827KP72Pm8OSeJU2fCd7A0X3rdGOADYIOIvJrjX9OBAZ6/BwDfFz88Lz31FKSnnzstPd093WGysrK4//77+emnn1i/fj1ffPHFOTczUUoVrkzJSB68siFzHu3CFY2qM3b2Zq5+bSGz1+8Ly6trfSnRdwTuALoZYxI9j2uBF4GrjDFJwFWe14GV39C+Pgz5m+3TTz+lRYsWtGzZkjvuuOOc/7333nu0a9eOli1bctNNN5HuOdh89dVXNGvWjJYtW54dJXLdunVccskltGrVihYtWpCUlJTn+pYtW0aDBg2oV68epUqV4tZbb+X77wN/rFQqFMVUjuKd29sw6Z5LKVUign99msDAj/5gW2r+NxQKRb70ulksIkZEWohIK89jhoikiUh3EWnoeT7oz4DzVKdO0aZ7ad26dTz//PPMnTuXVatWMW7cuHP+f+ONN/LHH3+watUqLrroIj744AMAnnnmGX7++WdWrVrF9OnTAZgwYQIPPvggiYmJJCQkEBMTk+c6d+/eTe3atc++jomJYffu3T59DqXCXaeGVfnpwct5+h8XsSL5ED1eX8iLP23kxOnwGCwtNAY1e/55iIo6d1pUlHu6D+bOnUvfvn3P3tWpSpUq5/x/7dq1XH755TRv3pzJkyezbt06ADp27MjAgQN57733yMpy1wt26NCBF154gZdeeonk5GTKls37Tjp5nVa6a8mUUr4oGRnB/11ejzlDu3B9q1pMWLCVbmPn833i7pCvzgmNRN+/P0ycCLGxYIz7eeJE93QfiEiBSXbgwIGMHz+eNWvWMHLkSE6dOgW4S+/PPfccO3fupFWrVqSlpdGvXz+mT59O2bJl6dGjB3Pnzs1zmTExMezcufPs6127dnHBBRf49DmUUn+pXqEMY25uyTeDL6N6hTI8OCWRf7671O+3K7WT0Ej04E7qO3aAy+V+9jHJA3Tv3p2pU6eSlpYGwMGD59ZCHTt2jJo1a3LmzBkm5+jhs3XrVi699FKeeeYZqlatys6dO9m2bRv16tVjyJAh9O7dm9WrV+e5znbt2pGUlMT27dvJyMhgypQp9O7d2+fPopQ6V5vYynx3f0f+d2NzkvYf47o3FzHi+7UcTs+wOjS/00HNCtC0aVOeeuopunTpQmRkJK1btyYuLu7s/5999lkuvfRSYmNjad68+dlx3h977DGSkpIQEbp3707Lli158cUXmTRpEiVLlqRGjRqMGDEiz3WWKFGC8ePH06NHD7Kysrj77rtp2tSaWykqFeoiIwy3XVKHa5vV5NXZm/hsaTI/rPqTx3o05p/tahMZIoOl6Xj0CtDtrRTAhj1HGTl9Hcu2H6RZrfMY3bsZbWLtOy6jt+PRh07VjVJK+eiimufx5aD2jLu1FanHTnPTO7/xyNRE9h87ZXVoPtGqG4ukpaXRvXv3v02fM2cO0dHRFkSklAJ3L7frW9XiyovOZ/y8Lby/aBuz1u3joSsbMuCyOEpGOq98rIneItHR0ZbcMlAp5Z1ypUswrGdjbmlbm9E/rOO5Hzcw5Y+djOrVlE4Nq1odXpE479CklFJBVLdqOT4a2I7372xLRqaL2z/4nXs/W87Og+mFv9kmtESvlFKFMMZwZZPz6dSwKu8v2sZb87Yyb9N+Bnetz71d6lOmZKTVIRZIS/RKKeWlMiUj+U8392BpVzY5n9d/SeLKVxcwc+1eW19dGzKJXoejV0oFywWVyvJWv4v5/F+XUq5UCe6dtJw7P1zGlv32HCwtJBJ9EIejD8p49HfffTfVq1enWbNmAV2PUso3l9Wvyo9DOjGyVxMSdx6m5+sLeWHGBo6dOmN1aOcIiUQfQsPRA+4xdGbOnGl1GEopL5SIjOCujnWZN7QrN15ci4kLt9Ft7AK+XbHLNtU5IZHoAzgcfdDHowfo3Lnz30bKVErZW9XypXm5b0u+u78jF1QswyNTV9F3whLW7j5idWihkegDNBy9JePRK6WcrVXtSky7ryMv39SCHQdO0Gv8Yp6atoZDJ6wbLC0kEn2AhqO3ZDx6pZTzRUQYbmlXm7lDuzKgQxxT/tjJFWPn89nSZLJcwa/OCYlEH6Dh6C0Zj14pFToqli3JqN5NmTHkchrXqMB/v1tLrzcX88eOwN94L6eQSPQQkOHoLRmPXikVehrVqMAX/2rP+H6tOZSewc0TlvDQlJXsOxqcwdJCJtEHQs7x6Fu2bMkjjzxyzv+zx6O/6qqraNy48dnpjz32GM2bN6dZs2Z07tyZli1b8uWXX9KsWTNatWrFxo0bufPOO/Nd72233UaHDh3YtGkTMTExZ+v+lVLOZYzhuhYXMOfRLvznigbMWLOXbmPmM33Vn4Fftx26/+h49NbT7a1UcCWnneDZ+A08dGVDmtWqWKxleDsevY51o5RSFoiNLsf7AwrN0X6hid4iOh69UipYbJ3oC+v14mR2Go/eDtV3SqnAsW1jbJkyZUhLS9MkFGAiQlpaGmXKlLE6FKVUgNi2RB8TE8OuXbtITU21OpSQV6ZMGb1SV6kQZttEX7JkSerWrWt1GEop5Xi2rbpRSinlH5rolVIqxGmiV0qpEGeLK2ONMalAcjHfXhU44MdwAs1J8TopVnBWvE6KFTTeQPIl1lgRqVbYTLZI9L4wxiR4cwmwXTgpXifFCs6K10mxgsYbSMGIVatulFIqxGmiV0qpEBcKiX6i1QEUkZPidVKs4Kx4nRQraLyBFPBYHV9Hr5RSqmChUKJXSilVAE30SikV4hyV6I0xZYwxy4wxq4wx64wxoz3T6xpjfjfGJBljvjTGlLI61mzGmEhjzEpjTLzntZ1j3WGMWWOMSTTGJHimVTHGzPbEO9sYU9nqOAGMMZWMMV8bYzYaYzYYYzrYONZGnm2a/ThqjHnIxvE+7Pl9rTXGfOH53dl5v33QE+s6Y8xDnmm22bbGmA+NMfuNMWtzTMszPuP2hjFmizFmtTHmYn/E4KhED5wGuolIS6AV0NMY0x54CXhNRBoCh4B7LIwxtweBDTle2zlWgCtEpFWOfr3DgTmeeOd4XtvBOGCmiDQGWuLexraMVUQ2ebZpK6ANkA5Mw4bxGmNqAUOAtiLSDIgEbsWm+60xphnwL+AS3PvBdcaYhthr234M9Mw1Lb/4rgEaeh6DgHf8EoGIOPIBRAErgEtxX1VWwjO9A/Cz1fF5YonxfIndgHjA2DVWTzw7gKq5pm0Canr+rglsskGc5wHb8XQmsHOsecR+NfCrXeMFagE7gSq4R7eNB3rYdb8Fbgbez/H6v8Djdtu2QBywNsfrPOMD3gVuy2s+Xx5OK9FnV4UkAvuB2cBW4LCIZHpm2YV7Z7WD13HvdC7P62jsGyuAALOMMcuNMYM8084XkT0AnufqlkX3l3pAKvCRp1rsfWNMOewZa263Al94/rZdvCKyGxgDpAB7gCPAcuy7364FOhtjoo0xUcC1QG1suG1zyS++7ANtNr9sa8clehHJEvcpcAzu07WL8potuFH9nTHmOmC/iCzPOTmPWS2PNYeOInIx7tPH+40xna0OKB8lgIuBd0SkNXACG1R7FMZTr90b+MrqWPLjqSu+HqgLXACUw70/5GaL/VZENuCuVpoNzARWAZkFvsneApIjHJfos4nIYWA+0B6oZIzJvolKDPCnVXHl0BHobYzZAUzBXX3zOvaMFQAR+dPzvB93HfIlwD5jTE0Az/N+6yI8axewS0R+97z+Gnfit2OsOV0DrBCRfZ7Xdoz3SmC7iKSKyBngW+Ay7L3ffiAiF4tIZ+AgkIQ9t21O+cW3C/cZSTa/bGtHJXpjTDVjTCXP32Vx75QbgHlAX89sA4DvrYnwLyLyhIjEiEgc7tP1uSLSHxvGCmCMKWeMqZD9N+665LXAdNxxgk3iFZG9wE5jTCPPpO7AemwYay638Ve1Ddgz3hSgvTEmyhhj+Gvb2nK/BTDGVPc81wFuxL2N7bhtc8ovvunAnZ7eN+2BI9lVPD6xujGliA0aLYCVwGrcSWiEZ3o9YBmwBfdpcWmrY80Vd1cg3s6xeuJa5XmsA57yTI/G3aCc5HmuYnWsnrhaAQmefeE7oLJdY/XEGwWkARVzTLNlvMBoYKPnN/YZUNqu+60n3kW4D0argO5227a4Dzx7gDO4S+z35Bcf7qqbt3C3Pa7B3fvJ5xh0CASllApxjqq6UUopVXSa6JVSKsRpoldKqRCniV4ppUKcJnqllApxmuiVUirEaaJXSqkQ9/94Q3oEGgzZUAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x28288469438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 4.绘制拟合曲线,该部分总是出现划线偏的问题,数据集随机没有处理好\n",
    "# %matplotlib inline\n",
    "# mydata = get_data('data.txt')\n",
    "logistic_model.cpu() \n",
    "w0, w1 = logistic_model.linear.weight[0]      # 模型得到的变量均为Variable类型\n",
    "w0, w1 = w0.data.numpy(), w1.data.numpy()\n",
    "b = logistic_model.linear.bias.data.numpy()\n",
    "plot_x = np.arange(30, 100, 0.1)\n",
    "plot_y = (-w0 * plot_x - b) / w1\n",
    "plt.plot(plot_x, plot_y, label='Fitting Line')\n",
    "mydata = get_data('data.txt')                # pyplot画的图可以直接叠加"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "小结:\n",
    "- random.shuffle()处理一维的列表和numpy数组时只是打乱顺序，但在处理多维numpy数组时，该函数会将数组的行随机的重复，所以要格外注意\n",
    "- 在PyTorch中，在计算数值时最好转换到相同的格式，如都是numpy格式或tensor格式，防止出错，cpu和gpu数据一定不要混用，训练时全放到GPU上，测试时可以使用cpu\n",
    "- 在PyTorch中，注意IntTensor()数据运算的'/'符号被认为整除符号，转换为float型计算除法\n",
    "- matplotlib绘制的图可以直接叠加，即使一部分图是在函数中绘制的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.原数据:\n",
      " [[ 1  3  2  0  5]\n",
      " [ 2  4  3  1  6]\n",
      " [ 0  2  1 -1  4]]\n",
      "2.按第3列排序:\n",
      " [[ 0  2  1 -1  4]\n",
      " [ 1  3  2  0  5]\n",
      " [ 2  4  3  1  6]]\n",
      "3.按第2行排序:\n",
      " [[ 0  1  2  3  5]\n",
      " [ 1  2  3  4  6]\n",
      " [-1  0  1  2  4]]\n"
     ]
    }
   ],
   "source": [
    "# numpy数组排序\n",
    "a = np.array([[1,3,2,0,5],[2, 4,3,1,6], [0, 2, 1, -1, 4]])\n",
    "data = np.array(a)\n",
    "print('1.原数据:\\n', data)\n",
    "print('2.按第3列排序:\\n', data[data[:,2].argsort()] )  # 按第3列排序\n",
    "print('3.按第2行排序:\\n', data[:,data[1].argsort()])   # 按第2行排列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 该方法可以将程序写到py脚本中！！！\n",
    "# %%writefile script_name.py\n",
    "# def hello():\n",
    "#     pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 多层全连接前向网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from torch import nn, optim\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置超参数(Hyperparameters)\n",
    "batch_size = 64\n",
    "learning_rate = 1e-2\n",
    "num_epochs = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.读取数据及预处理\n",
    "# Compose将各种预处理操作组合，ToTensor将图片处理为tensor，\n",
    "# Normalize(mean, variance)正则化\n",
    "data_tf = transforms.Compose([transforms.ToTensor(),\n",
    "                              transforms.Normalize([0.5], [0.5])])\n",
    "# 下载MNIST手写数字训练集\n",
    "train_dataset = datasets.MNIST(root='./data/MNIST', train=True, transform=data_tf, download=False)\n",
    "test_dataset = datasets.MNIST(root='./data/MNIST', train=False, transform=data_tf)\n",
    "# 创建数据迭代器便于训练模型\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)#, pin_memory=True)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.训练数据的维度:\n",
      "torch.Size([60000, 28, 28])\n",
      "torch.Size([60000])\n",
      "2.测试数据的维度:\n",
      "torch.Size([10000, 28, 28])\n",
      "torch.Size([10000])\n"
     ]
    }
   ],
   "source": [
    "print('1.训练数据的维度:')\n",
    "print(train_dataset.train_data.shape)\n",
    "print(train_dataset.train_labels.shape)\n",
    "print('2.测试数据的维度:')\n",
    "print(test_dataset.test_data.shape)\n",
    "print(test_dataset.test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([   2,    5,   14, ..., 9978, 9984, 9994], dtype=int64),)\n",
      "1.0~9样本的索引: [10, 5, 35, 30, 6, 15, 21, 17, 84, 9]\n",
      "2.手写体样本:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAADTCAYAAACRDeixAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGrtJREFUeJzt3We4VNXZxvE/NiICijUJxG5EosaaKKIIatRYgg1URCwoElEuijEarBiVRLEE7BjFroCKELFEo0iMij3YCxY0YkGxAVLeD+91nzWz51TOlL3X3L8vhylnZp995iye/axnPavF0qVLMTOz7Fuu0gdgZmbF4QHdzCwSHtDNzCLhAd3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCKxQjnfrEWLFlWxLHXp0qUtGvtcn5NCPie183kp5HOSzxG6mVkkPKCbmUXCA7qZWSQ8oJuZRcIDuplZJDygm5lFwgO6mVkkylqHXkpdunQB4MknnwRg0003BWDfffcFYJ999gFgypQped/373//G4AnnniiLMdp6bDxxhsDcNhhhwHQvXt3ANq3bw/AJptsAkByR6+vv/467/nPPvts6Q/WrJEcoZuZRaJFOfcULeaqrrZt2wJwyy23ACFi+v777wFYaaWVAGjdunW9r6Pnf/fddwAMGDAAgPHjxy/zsWVtpdt+++0HwKRJkwAYOHAgAFdddRUAixcvbvZ7pOWczJgxA4AtttgCgBVWWLaLVJ2Tr776qua+tdZaq0mv4ZWitavUZ2XttdcG4M477wTC1fs111wDwKxZs5r1+quuumrNv3fZZRcApk6dCsAPP/xQ7/d6paiZWZXJbIR+5ZVXAtC/f/9aH3/11VcB+PTTTwGYN29e8liAkFsX5Uh33nnnmvteeumlJh1bWqLRhqyxxhoAvPDCCwB06NAh7/FWrVoB4SqmOSp1Tg499FAArr76aiD8TMst9/+xzIcffgjAhAkTABg3bhwAb731Vq2v16dPHwBGjx5d8NiQIUMAuOyyyxp1bOWK0HOvHI444ggAevToAYTPucYB/V0kb0+cOBEIV8R33333shxKo5T7s9KuXTsA3njjDSBE0voZe/Xq1azX1+vlzrfod7LtttsCdX/exBG6mVmVyVyVyy9+8QsADj744Lz7FWkdeeSRQPgf78svvwTgm2++yXu+IrQzzzwTgOHDhwMhN3/WWWfVPLdfv34AzJ07t0g/RTooj5eMzG+77TYA5s+fX/ZjKpa+ffsCcO655wJhLkX50aeffhqAsWPHAoVXcHXRufn9738PQKdOnWoe07xN2uiqA+A3v/kNECLw5FdJ3lZEv+eeewKw/fbbA/Daa6+V4IhLb80116z59x133AHA6quvDsAVV1wBwEknnVSU99LYssEGG9Tcp8xCQ5F5UzlCNzOLhAd0M7NIZC7l0qZNGyBM6OnScOTIkQD861//atTrLFmyBICzzz4bCJfLw4YNA+CAAw6oee71118PFC5KyqqWLVsC8Kc//anWx2+66Sag8LI77Xr37l3z70suuQQIE1Ka9DzllFOAkKJrKqXwpk2bBuSnXNIqN72gVOOcOXMAeO6554AwAXjcccflfe96660HhL+3VVZZBYBBgwYBocw3a7bZZpuaf++66655jylN11xKDw8dOhTIn0hWmqfYHKGbmUUicxG6oku58cYbARgzZkyzXvf0008HQolS7gTGgQceCMQToWtRjUqmZNGiRQDcf//9ZT+m5jj66KOBsAAE4N133wXCgrOZM2cCDS/giNH5559f829Nil577bVAiNBF57Bjx45AKPfcaaed8p6X1clQLR466KCDCh479thjgVDqvKwUmT/88MN59+dG6CqPLjZH6GZmkchchD5ixIi820899VRRX/+BBx4A4IQTTqi5b4cddijqe1RabdEJwIMPPljmIykO5cmVH4ZQeqZFU8Wy8sorA7DhhhsC8O2339Y8phLItMmNDBtaEKQcueYcNttsMyDMpyiib+ziqbS5+OKLgbDACsKCn7vuuqso76HFWuussw4AN9xwAwA333xzUV6/Po7QzcwikYkIXdEQwE9/+lMgNEV6+eWXi/pejzzyCJAfocdGC4pk4cKFQN1VL2mnyCc3Z/n666+X5L06d+4MwB577AHk50K/+OKLkrxnOeh3f/jhhwOh/XRy4VFuPj6L9HOoyg3go48+AsLfQVPpqk3zcFp0pvc65phjlu1gl4EjdDOzSGQiQs/NdylaV45PLS6tYYou9VWUBy52vrlcPvvss7yvpaCNL7T0X9Uyp512WsnesxxU6aTa67qac6n6JcaNYNSgT3NIWmugBoB16dq1KxDq2JNzbc1pwb2sHKGbmUUiExG6WqBCyJ1ndZa9ktRQKamhSKSabb755gAMHjwYCCsmlUvO+rlTm+lXXnkFCCtfk6uEtXJ6r732AkLErnr0UrbTLSaNG926dau5T/NymlvSVcn+++9f72slr2bknXfeAUJOvZwcoZuZRSITEXouRQQx5vJKbbvttsu73dhcYTVSPbYiOuVJVYet7fmyTlsvavWwql3UMlq9XLQhg6LS8847L+/23nvvDYR1HGmlmvMtt9yy5r6tttoKCFcf6vejFaNajZ6knkcvvvhi3v2a13v77beLddiN5gjdzCwSqd6CTlFSbvWFKhl23HHHIh5ZoI0ztBEChDyjejQ0JG1b0HXp0gWAxx57DAgrKt977z0A1l9//VIfQurOSV2Skbn6xCxYsAAIvX7uu+++Zr9XmjeJVofGddddFwg5dPU1Up26InR1n0x2LlwWWfmsqOJOm1RonNImIM3tCZPLW9CZmVWZVOfQe/bsCcBGG21Uc18pa42h9pltdSHMKlVm5PY6AXjooYcqcTipVFdkrpyrKhaSHfRilazt19zB5ZdfDhT2RFH/kuOPPx7I73wZK21fqSzHqaeeChQ3Mm8qR+hmZpFIdYReTloxt++++xY8Vol60mJKbqit6hb1uq5mdUXms2fPBuCMM84Aqicyb6xkF0Z9VT17zA455BAgbEivfj6ff/55xY5JHKGbmUWi6iN0ReZDhgwBYLXVVgNg+vTpNc9Je21tXTp06ACEDnqi/TRnzJhR9mMqF1VnJOcNBg4cCIRe1fq62267ASEy33333QF44403Sn+wGXLyyScDYU9OVbkoWq2G9SGquZfJkycDhbs/VYIjdDOzSKQ6Qp81axZQmv33ll9+eQCGDRsGhPpiRWi6H7Jb5aKuisko9Z577qnE4ZRM69atgdCjHGDcuHEAtGrVqkmvpa6Ko0ePBuCiiy4CwmrATz75JO/52qMSQn+Xe++9FyhOrXqlaYWoukoOGjQICDlzVcGoDr0aKEJXl1JV/KSBI3Qzs0ikOkJ/9NFHgRA1A7Rt2xYIK9kaW5eu3g3aTUQ5wGR/E9XVFnuv0kpQ/bnoXMXSqbJHjx4ADB06FCjs814f9TOfN28eEKpYtBJSOXV91VXirbfeCoToW6sCIcxJaAecLOvYsSMAEydOBApXhqqnUmNXT8dAu5hp3mXOnDlAOnLn4gjdzCwSqY7Qa6P616lTpwLw8ccfN+r7tJtIXVHrpEmTAHjmmWeKcpxpkBs9Arz//vtA6CmfVS1btgTg9ttvB2DFFVds9Pcq76lVfclOkyeeeCIAF1xwARDq1Nu0aQNA//79Aejbty+Qv1N8DB0Y1UFQVz+ag1DOXH3PVdVSTRSh61xMmTIl73F9Rtq1aweEv7dycoRuZhYJD+hmZpHIRMpFTfcBhg8fDoRJzaZasmQJAF988QUAo0aNAuDCCy9sziGmilIQuU3NAObPnw+ECcGs2nrrrYHCcszaLFy4EIBLL70UCL/vuhoojRkzBgjtk4866igAunfvnvc8lfFlbWNtpZBU1ilKseic6u/kgw8+AMIWfFnZaq4cFi9eDEDv3r2BcI5mzpwJhLRcOTlCNzOLRKo3uKiNNnTVpKg28W3ItddeC8Dzzz8PlHYCq9IN+rVo6rrrrgNClKmorBKRQynOiaJkRZW5m2BrU5KRI0cCoSFZmlRig4sRI0YA4dwlNzrWbZUrDhgwACh92+pclf77qYuuxrRdX/LcjR07FgjnWFc3xeANLszMqkwmcui5tGgjd5NXy6fcnuYeFEFos4ZYqLTQGk+tChRdapNoLRRS+wLnygupsdu5554LwOOPPw6E0te5c+cCYd6mEhyhm5lFInM59CxIaw6wknxOClUih67WFv369QNCa+g0Xe34s1LIOXQzsyrjCL0EHGEU8jkpVIkIPQv8WSnkCN3MrMp4QDczi4QHdDOzSJQ1h25mZqXjCN3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCLhAd3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCLhAd3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCLhAd3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCLhAd3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCLhAd3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCLhAd3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCLhAd3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCLhAd3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCLhAd3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSHtDNzCKxQjnfrEWLFkvL+X6VsnTp0haNfa7PSSGfk9r5vBTyOcnnCN3MLBIe0M3MIuEB3cwsEh7Qzcwi4QHdzCwSZa1yscrYaqutABgxYgQAv/3tbwH47rvvAOjatSsAzz33XAWOzsyKxRG6mVkkWixdWr4yznLUjG677bYA9OjRA4CDDjoIgE033VTHAIB+bkWlr776KgDnn38+AK+99toyH0Pa6minTp0KwO677553/2effQbAQw89BECfPn1KdgxpOydp4Dr02vmzUsh16GZmVSZzOfTjjz8egI4dOwKw88475z2+zTbbACECT0bk11xzDQB33303AA8++GCJj7hyunXrBoRzIhdddBEA119/PQCrr756eQ/MMmullVYCYI899gBgypQpABx99NEAHHbYYQAsWLCg5nvuuuuuvOd+/vnn5TnYKuQI3cwsEpnLoS9ZsgQIEbcqNZTznjZtWt7tTz/9FAgReTlUOge4xhprAPD6668DsNpqqwEwefJkAA4++GAAFi1aVOy3rlOlz0kapTmHvuqqqwJw1FFHAeEzs/nmmwMwaNAgAMaNGwfAiy++CMBmm20GwIorrljwmqeccgoQrhDr4s9KIefQzcyqTOZy6BMnTgRCFYsi8e23375ix5Q2O+64IxAic7nwwguB8kbmpbD33nsDcM899wC1R4Py/fffAzBp0qS8+9977z0ALrvsMgB+/etfA6Hy54knnijiEafXmmuuCcC6664LhM9O7969Adh6660BeOyxxwDo1KkTAB9//HHe65x99tkA3HLLLUD+7+SHH36o9XuyZuONNwbCOTvggAMA2HXXXYGQPbjqqqsAmD59OgBvvfVW2Y7REbqZWSQ8oJuZRSJzKZcBAwYAYQHReuutB4RLxvfff78yB5YiWsqvkk2lJv7zn/9U7JiKSb/z+lItsvLKKwPQq1evWh8fPHhw3mvpsvmpp54CYPz48QC88sorAMyaNQsIE85ZoxLV7t27A/C3v/0NgB//+Md5z3v33XcB6Nu3LwB33nln3uP6bA0fPhyAc845J+9+TcDnPjZjxowi/RTloQnggQMHAnDggQcCIeVSF6XvlNrUZyU3jadJ5YULFxbxiB2hm5lFI3MRusoQtUDovPPOA8L/mtUcoa+99toA7LXXXkAo7dQkTSzGjh0LhMk2TVbV9rv/0Y9+BMDvfve7Wl9LZXZrrbUWAMst9/8xjiYH9VXmz58PwF//+lcAzjrrrGX8KSpj2LBhAJx22ml597/55psAXHfddQCMGjUKKJxA32WXXQC4+OKLAdhuu+0A+Prrr4EwyfznP/+55nt0ztJuyy23BODEE08EwlVd27Zt8543e/ZsIJRI62rmD3/4AwDPPvssAL/61a+AcFWkpngQyjyL/bfpCN3MLBKZi9BFkZRydoq0dDtJzbe0EClGRx55JBBKyxQ1xbbUWpG5IvXGuOSSS2q9X3lSLWWXww8/HAhzNaKIXzlQRbIAX331VaOPp5wUcUI4btHfRc+ePQH473//W+trKDK/7777gBC1vvPOO0Ao3fvggw+KdNTlc/XVVwOhDDGZI//nP/8JwMsvvwzA6aefDhReeXTu3BkI83xqraH21Z988knNc8eMGQPAhAkTgJB5aC5H6GZmkcjc0n/lOp9++mkgVLfU1YxLt7X0XwsfStkKoFJLlxWxarm28nTJ5lyVkLXl3IrE27dvD8Af//hHAI499ti852kOB+DMM89s0nuUeum/rj6eeeaZmvv0cyna1FXdCy+8UOtrqKJIz2/Tpg0Q8sZqjqe8cjGU8rOin1/5bgjzIBorFC1feeWVQJgv+fbbb+t97ZdeegkIDcr02VH76tqss846ee9ZFy/9NzOrMpnIoSsqh7AEWZF5coOK5JLt4447Dgi5UNWSKoLXTHQMOXYtiZfYqlvKSfnRt99+G4CRI0cCIULX/MQNN9xQ/oNrJH3mFZXm0t9FXZF5u3btALj99tuBEJlrGbtaMxczMi8H5frVKAxCZK6fRZviKAtQl+WXXx6An/3sZ0BoVPaPf/wDCOcw+T4AN910EwBffvll03+IejhCNzOLRCYidG0fl/tvNek65JBD6v1e1atr5vqII44AQnMv/S+slYB6veZsQVcpigBUAbTffvsBoU5blUCqh9XztDpSDau0mbQijsWLF5f82NMuWceuiFVtZQH+8pe/lPWYGvLwww8D4SoDYKONNgJCtcptt90GwBVXXAGEqFMRpOZfnn/+eQC6dOkCZPdKVj9fbZ9p1dxrpad+t9pMR9TwTX9P+qrGbsqLJ+VWuWjuRRVbxeII3cwsEpmrcik2bWmnnKJm9XPz0Vr51ViVquj46KOPgBAhNPS71byBIowkrSbULH9zZK3KRTbccEMgVAytssoqAMybNw+ADTbYoOa5c+fObdJrl2uDi6FDh9b8W5ugays5UcStSg7NW+mqbYcddgDgf//737IcQpOU8rOi3j633nprzX3aPL1Vq1ZAuHJN/v0oqleU3xBd+aqi7uSTT655rKmthF3lYmZWZao+Qhfl2FVFo23cIKz8amzteloidFViPPnkk0DIiSvX9/jjjwNhFaCuVrRiTlRXq81+l0VWI3StrEyuNFUvk9xqiaaqxBZ0P//5z4HQt0RzRppvUQSbpCsU5daVc1c+uZjK/VnRRjBaa7DTTjsBYYW1egS1bNkSgF/+8pdAqJCri6rMtLK0ORUtjtDNzKqMI/SEZDc5CHl15R8vvfTSel+jUtHoBRdcAIRVcOqc179//0Z9vyo3VPmzySabACFyaWhz3/pkLUJXZZCqO5Q7V45ZWx42pxoqDZtEqyeLVn6qM6By5aqU2WeffYAQpaqHyxlnnAHAvffeCzQ8b9MYaf+s6EpXFXOiK+IhQ4YAYY1CMarEHKGbmVWZTNShl5PyyrlVLsqrK0JtKEKvlGRXxaZunK0IQ6ttFaFXE82l6HetyFzUryWL6xRqoxy6InPRTkTJjpa6cnnkkUeAMK+kumr1Qc9KD/Sm0JXvoYceWuvjJ5xwAhBq+yvBEbqZWSQcoddBlSAQItbkirG0UX5XdbTaJ1N5zwULFtT7/erbvP/++wN195aPmeYLdA5EOWPtyBOLDz/8MO+26qPvuOOOWp+vXi7rr79+3vMU0Wv/2qau3Uizfv36AeFnXGGF/GFz5syZQFi9XkmO0M3MIuEIvQ650bj6vqjfS1qpf7PqY/v06QOEnd1VU52sHVbnSu2iohp8VSwUazeVNFNedPDgwXn366pHnwGt/ovFT37yk1pvJ1eSJuk8aMWl+p6MHj0aKNyLNYv0d6SKt9atW+c9/s033wAhd97QFXA5OEI3M4tE5iJ0RVCKGm+++eaivr5qznN3LVePh4Y6O6aF6mDVo+KYY47Je3z8+PFAqOC4/PLLgRCdKY/697//HYAbb7yxxEdcOV27dgXCvpLJeQPt/lTXXptZ19x+3MndsJRDj4FWz2p9huiqTfMs06dPL++B1cMRuplZJDKxUjS3t4iiS/U5V5+VxlIXuWS/Et1WxDFnzpyax1R7nPZeLkn6WbSKL5kvTe6/qt3N1WVRu0EVQ1rOiah/h3apT9abaz5BVzvF7lsN6Vgpqkoo7YepuSP1f580aVJdxwLA/fffD4S9RdVrX2s3lkWlPyuKyFXppnMkGnuUOy8HrxQ1M6syHtDNzCKRuZSLivdVNqXl7rpfl4K6dNRlk8rOkmkG3dZmDw888AAQGnHlvkZjVfqSMUkLhrS1nNoaqM2BLpu1aGbhwoVFP4a0nBMtujrppJOAwra4WhDTuXNnoDSpFklDykW0ebKW9KtlbHJjC50/LbI555xzAJg8eTIQJhKbo1KfFZUlaixo37593uNKS+mclLO9gVMuZmZVJhMReq4999wTCBG3KIrXpKcWASmC1/+6iraTE5xqtlSMzW/TEo2mSVrOiSJvtXNI6tmzJxAm30spTRG69OrVCwibM6isUbe1+YMi8TfffBOAbt26ATB79uxmH0OlPisqQ6yrFfBuu+0GwKOPPlqst2w0R+hmZlUmcxF6FqQlGk2TSp+T5EYO7dq103sBMG3aNAC6d+8OwKJFi4p9CAXSGKGL2umOHDkSgE6dOuU9rs0blDtv6qbH9anUZ0Xb7G2xxRZ592uT9FNPPbVYb9VkjtDNzKqMI/QSqHQ0mkaVPieaY5kwYULe/cqlayPsYuSAGyvNEXolVeqzokVmHTp0AMLiQlWJFfMqpKkcoZuZVZnMNecyWxaqelI9taozevfuDZQ3Mrd0GjVqVN5XrduoZGTeVI7Qzcwi4Rx6CVQ6X5xGPieFnEOvnT8rhZxDNzOrMmWN0M3MrHQcoZuZRcIDuplZJDygm5lFwgO6mVkkPKCbmUXCA7qZWSQ8oJuZRcIDuplZJDygm5lFwgO6mVkkPKCbmUXCA7qZWSQ8oJuZRcIDuplZJDygm5lFwgO6mVkkPKCbmUXCA7qZWSQ8oJuZRcIDuplZJDygm5lFwgO6mVkkPKCbmUXi/wB1Un0FtXeAzAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x282c1a97080>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 显示数据\n",
    "%matplotlib inline\n",
    "test = test_dataset.test_labels.numpy()     # 将tensor数据转换为numpy数据\n",
    "print(np.where(test==1))\n",
    "sample_index = [np.where(test==i)[0][1] for i in range(10)] # np.where的输出(array([], dtype),)\n",
    "print('1.0~9样本的索引:', sample_index)\n",
    "sample_data = test_dataset.test_data[sample_index]    # 获取10个样本数据\n",
    "print('2.手写体样本:')\n",
    "for i in range(10):\n",
    "    plt.subplot(2,5,i+1)\n",
    "    sample_data = test_dataset.test_data[sample_index][i]\n",
    "    plt.axis(\"off\")       # 关掉坐标轴\n",
    "    plt.imshow(sample_data, interpolation='none', cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2.定义模型\n",
    "# (1)定义简单的模型\n",
    "class Simple_Net(nn.Module):\n",
    "    def __init__(self, in_dim, n_hidden_1, n_hidden_2, out_dim):\n",
    "        super().__init__() \n",
    "        self.layer1 = nn.Linear(in_dim, n_hidden_1)      # 定义结构\n",
    "        self.layer2 = nn.Linear(n_hidden_1, n_hidden_2)\n",
    "        self.layer3 = nn.Linear(n_hidden_2, out_dim)\n",
    "    def forward(self, x):    # 前向传播\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        return x\n",
    "# (2)定义含激活函数的模型\n",
    "class Activation_Net(nn.Module):\n",
    "    def __init__(self, in_dim, n_hidden_1, n_hidden_2, out_dim):\n",
    "        super().__init__()    # 定义结构\n",
    "        self.layer1 = nn.Sequential(nn.Linear(in_dim, n_hidden_1),\n",
    "                                    nn.ReLU(True))  \n",
    "        self.layer2 = nn.Sequential(nn.Linear(n_hidden_1, n_hidden_2),\n",
    "                                    nn.ReLU(True))\n",
    "        self.layer3 = nn.Linear(n_hidden_2, out_dim)\n",
    "    def forward(self, x):    # 前向传播\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        return x\n",
    "# (3)定义含批标准化的模型\n",
    "class Batch_Net(nn.Module):\n",
    "    def __init__(self, in_dim, n_hidden_1, n_hidden_2, out_dim):\n",
    "        super().__init__()    # 定义结构\n",
    "        # Sequential组合网络的层\n",
    "        self.layer1 = nn.Sequential(nn.Linear(in_dim, n_hidden_1),\n",
    "                                    nn.BatchNorm1d(n_hidden_1), nn.ReLU(True))  \n",
    "        self.layer2 = nn.Sequential(nn.Linear(n_hidden_1, n_hidden_2),\n",
    "                                    nn.BatchNorm1d(n_hidden_2), nn.ReLU(True))\n",
    "        self.layer3 = nn.Linear(n_hidden_2, out_dim)\n",
    "    def forward(self, x):    # 前向传播\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3.定义损失函数及优化方法\n",
    "# model = Simple_Net(784, 300, 100, 10)    # Activation_Net, Batch_Net\n",
    "# model = Activation_Net(784, 300, 100, 10) \n",
    "model = Batch_Net(784, 300, 100, 10) \n",
    "# if torch.cuda.is_available():\n",
    "#     model = model.cuda()\n",
    "model.to(device)\n",
    "criterion = nn.CrossEntropyLoss()       # 使用交叉熵损失函数\n",
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Epoch 0\n",
      "Num batches 450\n",
      "loss 0.5349\n",
      "Num batches 900\n",
      "loss 0.3815\n",
      "--------------------------------------------------\n",
      "Epoch 1\n",
      "Num batches 450\n",
      "loss 0.2376\n",
      "Num batches 900\n",
      "loss 0.1064\n",
      "--------------------------------------------------\n",
      "Epoch 2\n",
      "Num batches 450\n",
      "loss 0.1314\n",
      "Num batches 900\n",
      "loss 0.1567\n",
      "--------------------------------------------------\n",
      "Epoch 3\n",
      "Num batches 450\n",
      "loss 0.0243\n",
      "Num batches 900\n",
      "loss 0.1633\n",
      "--------------------------------------------------\n",
      "Epoch 4\n",
      "Num batches 450\n",
      "loss 0.0927\n",
      "Num batches 900\n",
      "loss 0.0463\n",
      "--------------------------------------------------\n",
      "Epoch 5\n",
      "Num batches 450\n",
      "loss 0.0887\n",
      "Num batches 900\n",
      "loss 0.0996\n",
      "--------------------------------------------------\n",
      "Epoch 6\n",
      "Num batches 450\n",
      "loss 0.1338\n",
      "Num batches 900\n",
      "loss 0.0303\n",
      "--------------------------------------------------\n",
      "Epoch 7\n",
      "Num batches 450\n",
      "loss 0.0633\n",
      "Num batches 900\n",
      "loss 0.0630\n",
      "--------------------------------------------------\n",
      "Epoch 8\n",
      "Num batches 450\n",
      "loss 0.0460\n",
      "Num batches 900\n",
      "loss 0.0108\n",
      "--------------------------------------------------\n",
      "Epoch 9\n",
      "Num batches 450\n",
      "loss 0.0281\n",
      "Num batches 900\n",
      "loss 0.0765\n",
      "Wall time: 51.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 4.训练模型\n",
    "# 批次数据x:[64, 1, 28, 28], y:[64]\n",
    "epochs = 10\n",
    "for epoch in range(epochs):\n",
    "    print('-'*50)\n",
    "    print('Epoch {}'.format(epoch))\n",
    "    for num_batches, data in enumerate(train_loader):   # 访问一个完整的epoch\n",
    "        x_train, y_train = data               # 获取训练批次数据\n",
    "        x_train, y_train = x_train.to(device), y_train.to(device)\n",
    "#         print(x_train.device)\n",
    "#         print(x_train.shape, y_train.shape)\n",
    "        # 前向传播\n",
    "        # 注意：该处数据的维度应为[-1, input_dim]，第一个维度前面都是batch_size，\n",
    "        # 但数据集尾部会出现不足batch_size的部分，-1更合理，第2个维度是固定的\n",
    "        x_train = x_train.reshape([-1, 28*28]) # 使维度变为[-1, 28*28]\n",
    "#         print(x_train.shape)\n",
    "        out = model(x_train)                  # 计算模型输出 [64,10]\n",
    "#         print(out.shape)\n",
    "#         print(y_train.shape, y_train)\n",
    "        \n",
    "        loss = criterion(out, y_train)        # 计算损失函数\n",
    "        print_loss = loss.data\n",
    "        # 反向传播\n",
    "        optimizer.zero_grad()        # 梯度归零\n",
    "        loss.backward()              # 梯度方向传播\n",
    "        optimizer.step()             # 更新参数\n",
    "        if (num_batches + 1) % 450 == 0:\n",
    "            print('Num batches {}'.format(num_batches + 1))\n",
    "            print('loss {:.4f}'.format(print_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 Test Loss: 0.020145, ACC: 1.000000\n",
      "Epoch: 4 Test Loss: 0.115917, ACC: 0.984375\n",
      "Epoch: 6 Test Loss: 0.082434, ACC: 0.984375\n",
      "Epoch: 8 Test Loss: 0.047114, ACC: 0.984375\n",
      "Epoch: 10 Test Loss: 0.124163, ACC: 0.953125\n",
      "Epoch: 12 Test Loss: 0.041718, ACC: 0.984375\n",
      "Epoch: 14 Test Loss: 0.046981, ACC: 1.000000\n",
      "Epoch: 16 Test Loss: 0.127097, ACC: 0.968750\n",
      "Epoch: 18 Test Loss: 0.088028, ACC: 0.968750\n",
      "Epoch: 20 Test Loss: 0.353063, ACC: 0.921875\n",
      "Epoch: 22 Test Loss: 0.060695, ACC: 0.984375\n",
      "Epoch: 24 Test Loss: 0.177051, ACC: 0.937500\n",
      "Epoch: 26 Test Loss: 0.066317, ACC: 0.968750\n",
      "Epoch: 28 Test Loss: 0.112695, ACC: 0.968750\n",
      "Epoch: 30 Test Loss: 0.125585, ACC: 0.953125\n",
      "Epoch: 32 Test Loss: 0.256382, ACC: 0.921875\n",
      "Epoch: 34 Test Loss: 0.126841, ACC: 0.953125\n",
      "Epoch: 36 Test Loss: 0.049064, ACC: 0.984375\n",
      "Epoch: 38 Test Loss: 0.152388, ACC: 0.937500\n",
      "Epoch: 40 Test Loss: 0.038084, ACC: 1.000000\n",
      "Epoch: 42 Test Loss: 0.147117, ACC: 0.968750\n",
      "Epoch: 44 Test Loss: 0.080607, ACC: 0.953125\n",
      "Epoch: 46 Test Loss: 0.165979, ACC: 0.921875\n",
      "Epoch: 48 Test Loss: 0.035457, ACC: 0.984375\n",
      "Epoch: 50 Test Loss: 0.017736, ACC: 1.000000\n",
      "Epoch: 52 Test Loss: 0.033042, ACC: 1.000000\n",
      "Epoch: 54 Test Loss: 0.085118, ACC: 0.984375\n",
      "Epoch: 56 Test Loss: 0.298410, ACC: 0.937500\n",
      "Epoch: 58 Test Loss: 0.056818, ACC: 0.984375\n",
      "Epoch: 60 Test Loss: 0.158191, ACC: 0.953125\n",
      "Epoch: 62 Test Loss: 0.137956, ACC: 0.968750\n",
      "Epoch: 64 Test Loss: 0.190920, ACC: 0.937500\n",
      "Epoch: 66 Test Loss: 0.132045, ACC: 0.968750\n",
      "Epoch: 68 Test Loss: 0.080324, ACC: 0.984375\n",
      "Epoch: 70 Test Loss: 0.089121, ACC: 0.968750\n",
      "Epoch: 72 Test Loss: 0.101691, ACC: 0.968750\n",
      "Epoch: 74 Test Loss: 0.053264, ACC: 0.984375\n",
      "Epoch: 76 Test Loss: 0.152290, ACC: 0.937500\n",
      "Epoch: 78 Test Loss: 0.048358, ACC: 1.000000\n",
      "Epoch: 80 Test Loss: 0.045363, ACC: 0.984375\n",
      "Epoch: 82 Test Loss: 0.020570, ACC: 1.000000\n",
      "Epoch: 84 Test Loss: 0.052771, ACC: 0.984375\n",
      "Epoch: 86 Test Loss: 0.038789, ACC: 0.984375\n",
      "Epoch: 88 Test Loss: 0.034560, ACC: 0.984375\n",
      "Epoch: 90 Test Loss: 0.150779, ACC: 0.968750\n",
      "Epoch: 92 Test Loss: 0.098422, ACC: 0.968750\n",
      "Epoch: 94 Test Loss: 0.211153, ACC: 0.937500\n",
      "Epoch: 96 Test Loss: 0.003612, ACC: 1.000000\n",
      "Epoch: 98 Test Loss: 0.002422, ACC: 1.000000\n",
      "Epoch: 100 Test Loss: 0.008612, ACC: 1.000000\n",
      "Epoch: 102 Test Loss: 0.005833, ACC: 1.000000\n",
      "Epoch: 104 Test Loss: 0.174515, ACC: 0.953125\n",
      "Epoch: 106 Test Loss: 0.063330, ACC: 0.984375\n",
      "Epoch: 108 Test Loss: 0.004965, ACC: 1.000000\n",
      "Epoch: 110 Test Loss: 0.004216, ACC: 1.000000\n",
      "Epoch: 112 Test Loss: 0.004999, ACC: 1.000000\n",
      "Epoch: 114 Test Loss: 0.046828, ACC: 0.984375\n",
      "Epoch: 116 Test Loss: 0.005379, ACC: 1.000000\n",
      "Epoch: 118 Test Loss: 0.022416, ACC: 1.000000\n",
      "Epoch: 120 Test Loss: 0.005737, ACC: 1.000000\n",
      "Epoch: 122 Test Loss: 0.017493, ACC: 0.984375\n",
      "Epoch: 124 Test Loss: 0.059818, ACC: 0.984375\n",
      "Epoch: 126 Test Loss: 0.032127, ACC: 0.984375\n",
      "Epoch: 128 Test Loss: 0.007405, ACC: 1.000000\n",
      "Epoch: 130 Test Loss: 0.072337, ACC: 0.968750\n",
      "Epoch: 132 Test Loss: 0.049138, ACC: 0.984375\n",
      "Epoch: 134 Test Loss: 0.088462, ACC: 0.937500\n",
      "Epoch: 136 Test Loss: 0.003862, ACC: 1.000000\n",
      "Epoch: 138 Test Loss: 0.002715, ACC: 1.000000\n",
      "Epoch: 140 Test Loss: 0.005248, ACC: 1.000000\n",
      "Epoch: 142 Test Loss: 0.107714, ACC: 0.984375\n",
      "Epoch: 144 Test Loss: 0.006526, ACC: 1.000000\n",
      "Epoch: 146 Test Loss: 0.015501, ACC: 1.000000\n",
      "Epoch: 148 Test Loss: 0.017030, ACC: 0.984375\n",
      "Epoch: 150 Test Loss: 0.037339, ACC: 0.984375\n",
      "Epoch: 152 Test Loss: 0.109095, ACC: 0.937500\n",
      "Epoch: 154 Test Loss: 0.073880, ACC: 0.953125\n",
      "Epoch: 156 Test Loss: 0.066348, ACC: 0.984375\n",
      "----------Test Done!----------\n",
      "Epoch: 157 Total_Loss: 0.070033 Total_ACC: 0.979000\n",
      "Wall time: 1.17 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 5.评价模型的性能\n",
    "model.eval()                                      # 将模型转换为测试状态\n",
    "total_loss = 0\n",
    "total_acc = 0\n",
    "test_epoch = 0 \n",
    "\n",
    "for data in test_loader:\n",
    "    eval_loss = 0\n",
    "    eval_acc = 0\n",
    "    img, label = data                                      # 获取测试数据\n",
    "#     print(img.shape)\n",
    "    img = img.view(img.size(0), -1)                        # 返回一个数据相同但维度不同的tensor\n",
    "    # 由于测试状态下不需方向传播，所以可以释放缓存\n",
    "    with torch.no_grad():\n",
    "        img, label = img.to(device), label.to(device)\n",
    "#         img, label = (Variable(img).cuda(), Variable(label).cuda()) \\\n",
    "#                      if torch.cuda.is_available() \\\n",
    "#                      else (Variable(img), Variable(label))\n",
    "        out = model(img)     # 计算图像的标签 [img.size(0), 10]\n",
    "        loss = criterion(out, label)\n",
    "        eval_loss += loss.data* label.size(0)              # 计算该批次的总损失\n",
    "        _, pred = torch.max(out, 1)                        # 在一行内按列比大小得[64,1],tenso为[64]\n",
    "    #     print(_, pred.shape)\n",
    "        num_correct = (pred == label).sum()\n",
    "        eval_acc += num_correct.data\n",
    "    #     print(eval_acc)    \n",
    "        total_loss += eval_loss                            # 累计所有批次数据的损失和准确率\n",
    "        total_acc  += eval_acc\n",
    "        if (test_epoch + 1) % 2 == 0:\n",
    "            batch_loss  = (eval_loss / img.size(0))        # 计算该批次的损失\n",
    "            batch_acc   = (eval_acc.float() / img.size(0)) # 计算该批次的准确率\n",
    "            print('Epoch: {} Test Loss: {:.6f}, ACC: {:.6f}'.format(test_epoch+1, batch_loss, batch_acc))\n",
    "        test_epoch += 1\n",
    "print('----------Test Done!----------')\n",
    "print('Epoch: {} Total_Loss: {:.6f} Total_ACC: {:.6f}'.format(\\\n",
    "      test_epoch, total_loss/len(test_dataset), total_acc.float()/len(test_dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.模型一\n",
    "\n",
    "Epoch: 157 Total_Loss: 0.291483 Total_ACC: 0.919000\n",
    "\n",
    "\n",
    "2.模型二\n",
    "\n",
    "Epoch: 157 Total_Loss: 0.195421 Total_ACC: 0.944600\n",
    "\n",
    "3.模型三\n",
    "\n",
    "Epoch: 157 Total_Loss: 0.070033 Total_ACC: 0.979000"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
